drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint8_t optional;
 104         /**< optional expand field. Default 0 to expand, 1 not go deeper. */
 105 };
 106
 107 /** Object returned by mlx5_flow_expand_rss(). */
 108 struct mlx5_flow_expand_rss {
 109         uint32_t entries;
 110         /**< Number of entries @p patterns and @p priorities. */
 111         struct {
 112                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 113                 uint32_t priority; /**< Priority offset for each expansion. */
 114         } entry[];
 115 };
 116
 117 static void
 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 119
 120 static bool
 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 122 {
 123         switch (item->type) {
 124         case RTE_FLOW_ITEM_TYPE_ETH:
 125         case RTE_FLOW_ITEM_TYPE_VLAN:
 126         case RTE_FLOW_ITEM_TYPE_IPV4:
 127         case RTE_FLOW_ITEM_TYPE_IPV6:
 128         case RTE_FLOW_ITEM_TYPE_UDP:
 129         case RTE_FLOW_ITEM_TYPE_TCP:
 130         case RTE_FLOW_ITEM_TYPE_VXLAN:
 131         case RTE_FLOW_ITEM_TYPE_NVGRE:
 132         case RTE_FLOW_ITEM_TYPE_GRE:
 133         case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 return true;
 135         default:
 136                 break;
 137         }
 138         return false;
 139 }
 140
 141 static enum rte_flow_item_type
 142 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 143 {
 144         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 145         uint16_t ether_type = 0;
 146         uint16_t ether_type_m;
 147         uint8_t ip_next_proto = 0;
 148         uint8_t ip_next_proto_m;
 149
 150         if (item == NULL || item->spec == NULL)
 151                 return ret;
 152         switch (item->type) {
 153         case RTE_FLOW_ITEM_TYPE_ETH:
 154                 if (item->mask)
 155                         ether_type_m = ((const struct rte_flow_item_eth *)
 156                                                 (item->mask))->type;
 157                 else
 158                         ether_type_m = rte_flow_item_eth_mask.type;
 159                 if (ether_type_m != RTE_BE16(0xFFFF))
 160                         break;
 161                 ether_type = ((const struct rte_flow_item_eth *)
 162                                 (item->spec))->type;
 163                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 164                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 165                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 166                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 167                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 168                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 169                 else
 170                         ret = RTE_FLOW_ITEM_TYPE_END;
 171                 break;
 172         case RTE_FLOW_ITEM_TYPE_VLAN:
 173                 if (item->mask)
 174                         ether_type_m = ((const struct rte_flow_item_vlan *)
 175                                                 (item->mask))->inner_type;
 176                 else
 177                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 178                 if (ether_type_m != RTE_BE16(0xFFFF))
 179                         break;
 180                 ether_type = ((const struct rte_flow_item_vlan *)
 181                                 (item->spec))->inner_type;
 182                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 183                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 185                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 186                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 187                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 188                 else
 189                         ret = RTE_FLOW_ITEM_TYPE_END;
 190                 break;
 191         case RTE_FLOW_ITEM_TYPE_IPV4:
 192                 if (item->mask)
 193                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 194                                         (item->mask))->hdr.next_proto_id;
 195                 else
 196                         ip_next_proto_m =
 197                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 198                 if (ip_next_proto_m != 0xFF)
 199                         break;
 200                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 201                                 (item->spec))->hdr.next_proto_id;
 202                 if (ip_next_proto == IPPROTO_UDP)
 203                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 204                 else if (ip_next_proto == IPPROTO_TCP)
 205                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 206                 else if (ip_next_proto == IPPROTO_IP)
 207                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 208                 else if (ip_next_proto == IPPROTO_IPV6)
 209                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 210                 else
 211                         ret = RTE_FLOW_ITEM_TYPE_END;
 212                 break;
 213         case RTE_FLOW_ITEM_TYPE_IPV6:
 214                 if (item->mask)
 215                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 216                                                 (item->mask))->hdr.proto;
 217                 else
 218                         ip_next_proto_m =
 219                                 rte_flow_item_ipv6_mask.hdr.proto;
 220                 if (ip_next_proto_m != 0xFF)
 221                         break;
 222                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 223                                 (item->spec))->hdr.proto;
 224                 if (ip_next_proto == IPPROTO_UDP)
 225                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 226                 else if (ip_next_proto == IPPROTO_TCP)
 227                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 228                 else if (ip_next_proto == IPPROTO_IP)
 229                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 230                 else if (ip_next_proto == IPPROTO_IPV6)
 231                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 232                 else
 233                         ret = RTE_FLOW_ITEM_TYPE_END;
 234                 break;
 235         default:
 236                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 237                 break;
 238         }
 239         return ret;
 240 }
 241
 242 #define MLX5_RSS_EXP_ELT_N 16
 243
 244 /**
 245  * Expand RSS flows into several possible flows according to the RSS hash
 246  * fields requested and the driver capabilities.
 247  *
 248  * @param[out] buf
 249  *   Buffer to store the result expansion.
 250  * @param[in] size
 251  *   Buffer size in bytes. If 0, @p buf can be NULL.
 252  * @param[in] pattern
 253  *   User flow pattern.
 254  * @param[in] types
 255  *   RSS types to expand (see ETH_RSS_* definitions).
 256  * @param[in] graph
 257  *   Input graph to expand @p pattern according to @p types.
 258  * @param[in] graph_root_index
 259  *   Index of root node in @p graph, typically 0.
 260  *
 261  * @return
 262  *   A positive value representing the size of @p buf in bytes regardless of
 263  *   @p size on success, a negative errno value otherwise and rte_errno is
 264  *   set, the following errors are defined:
 265  *
 266  *   -E2BIG: graph-depth @p graph is too deep.
 267  *   -EINVAL: @p size has not enough space for expanded pattern.
 268  */
 269 static int
 270 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 271                      const struct rte_flow_item *pattern, uint64_t types,
 272                      const struct mlx5_flow_expand_node graph[],
 273                      int graph_root_index)
 274 {
 275         const struct rte_flow_item *item;
 276         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 277         const int *next_node;
 278         const int *stack[MLX5_RSS_EXP_ELT_N];
 279         int stack_pos = 0;
 280         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 281         unsigned int i;
 282         size_t lsize;
 283         size_t user_pattern_size = 0;
 284         void *addr = NULL;
 285         const struct mlx5_flow_expand_node *next = NULL;
 286         struct rte_flow_item missed_item;
 287         int missed = 0;
 288         int elt = 0;
 289         const struct rte_flow_item *last_item = NULL;
 290
 291         memset(&missed_item, 0, sizeof(missed_item));
 292         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 293                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 294         if (lsize > size)
 295                 return -EINVAL;
 296         buf->entry[0].priority = 0;
 297         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 298         buf->entries = 0;
 299         addr = buf->entry[0].pattern;
 300         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 301                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 302                         user_pattern_size += sizeof(*item);
 303                         continue;
 304                 }
 305                 last_item = item;
 306                 for (i = 0; node->next && node->next[i]; ++i) {
 307                         next = &graph[node->next[i]];
 308                         if (next->type == item->type)
 309                                 break;
 310                 }
 311                 if (next)
 312                         node = next;
 313                 user_pattern_size += sizeof(*item);
 314         }
 315         user_pattern_size += sizeof(*item); /* Handle END item. */
 316         lsize += user_pattern_size;
 317         if (lsize > size)
 318                 return -EINVAL;
 319         /* Copy the user pattern in the first entry of the buffer. */
 320         rte_memcpy(addr, pattern, user_pattern_size);
 321         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 322         buf->entries = 1;
 323         /* Start expanding. */
 324         memset(flow_items, 0, sizeof(flow_items));
 325         user_pattern_size -= sizeof(*item);
 326         /*
 327          * Check if the last valid item has spec set, need complete pattern,
 328          * and the pattern can be used for expansion.
 329          */
 330         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 331         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 332                 /* Item type END indicates expansion is not required. */
 333                 return lsize;
 334         }
 335         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 336                 next = NULL;
 337                 missed = 1;
 338                 for (i = 0; node->next && node->next[i]; ++i) {
 339                         next = &graph[node->next[i]];
 340                         if (next->type == missed_item.type) {
 341                                 flow_items[0].type = missed_item.type;
 342                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 343                                 break;
 344                         }
 345                         next = NULL;
 346                 }
 347         }
 348         if (next && missed) {
 349                 elt = 2; /* missed item + item end. */
 350                 node = next;
 351                 lsize += elt * sizeof(*item) + user_pattern_size;
 352                 if (lsize > size)
 353                         return -EINVAL;
 354                 if (node->rss_types & types) {
 355                         buf->entry[buf->entries].priority = 1;
 356                         buf->entry[buf->entries].pattern = addr;
 357                         buf->entries++;
 358                         rte_memcpy(addr, buf->entry[0].pattern,
 359                                    user_pattern_size);
 360                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 361                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 362                         addr = (void *)(((uintptr_t)addr) +
 363                                         elt * sizeof(*item));
 364                 }
 365         }
 366         memset(flow_items, 0, sizeof(flow_items));
 367         next_node = node->next;
 368         stack[stack_pos] = next_node;
 369         node = next_node ? &graph[*next_node] : NULL;
 370         while (node) {
 371                 flow_items[stack_pos].type = node->type;
 372                 if (node->rss_types & types) {
 373                         size_t n;
 374                         /*
 375                          * compute the number of items to copy from the
 376                          * expansion and copy it.
 377                          * When the stack_pos is 0, there are 1 element in it,
 378                          * plus the addition END item.
 379                          */
 380                         elt = stack_pos + 2;
 381                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 382                         lsize += elt * sizeof(*item) + user_pattern_size;
 383                         if (lsize > size)
 384                                 return -EINVAL;
 385                         n = elt * sizeof(*item);
 386                         buf->entry[buf->entries].priority =
 387                                 stack_pos + 1 + missed;
 388                         buf->entry[buf->entries].pattern = addr;
 389                         buf->entries++;
 390                         rte_memcpy(addr, buf->entry[0].pattern,
 391                                    user_pattern_size);
 392                         addr = (void *)(((uintptr_t)addr) +
 393                                         user_pattern_size);
 394                         rte_memcpy(addr, &missed_item,
 395                                    missed * sizeof(*item));
 396                         addr = (void *)(((uintptr_t)addr) +
 397                                 missed * sizeof(*item));
 398                         rte_memcpy(addr, flow_items, n);
 399                         addr = (void *)(((uintptr_t)addr) + n);
 400                 }
 401                 /* Go deeper. */
 402                 if (!node->optional && node->next) {
 403                         next_node = node->next;
 404                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 405                                 rte_errno = E2BIG;
 406                                 return -rte_errno;
 407                         }
 408                         stack[stack_pos] = next_node;
 409                 } else if (*(next_node + 1)) {
 410                         /* Follow up with the next possibility. */
 411                         ++next_node;
 412                 } else {
 413                         /* Move to the next path. */
 414                         if (stack_pos)
 415                                 next_node = stack[--stack_pos];
 416                         next_node++;
 417                         stack[stack_pos] = next_node;
 418                 }
 419                 node = *next_node ? &graph[*next_node] : NULL;
 420         };
 421         return lsize;
 422 }
 423
 424 enum mlx5_expansion {
 425         MLX5_EXPANSION_ROOT,
 426         MLX5_EXPANSION_ROOT_OUTER,
 427         MLX5_EXPANSION_ROOT_ETH_VLAN,
 428         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 429         MLX5_EXPANSION_OUTER_ETH,
 430         MLX5_EXPANSION_OUTER_ETH_VLAN,
 431         MLX5_EXPANSION_OUTER_VLAN,
 432         MLX5_EXPANSION_OUTER_IPV4,
 433         MLX5_EXPANSION_OUTER_IPV4_UDP,
 434         MLX5_EXPANSION_OUTER_IPV4_TCP,
 435         MLX5_EXPANSION_OUTER_IPV6,
 436         MLX5_EXPANSION_OUTER_IPV6_UDP,
 437         MLX5_EXPANSION_OUTER_IPV6_TCP,
 438         MLX5_EXPANSION_VXLAN,
 439         MLX5_EXPANSION_VXLAN_GPE,
 440         MLX5_EXPANSION_GRE,
 441         MLX5_EXPANSION_NVGRE,
 442         MLX5_EXPANSION_GRE_KEY,
 443         MLX5_EXPANSION_MPLS,
 444         MLX5_EXPANSION_ETH,
 445         MLX5_EXPANSION_ETH_VLAN,
 446         MLX5_EXPANSION_VLAN,
 447         MLX5_EXPANSION_IPV4,
 448         MLX5_EXPANSION_IPV4_UDP,
 449         MLX5_EXPANSION_IPV4_TCP,
 450         MLX5_EXPANSION_IPV6,
 451         MLX5_EXPANSION_IPV6_UDP,
 452         MLX5_EXPANSION_IPV6_TCP,
 453 };
 454
 455 /** Supported expansion of items. */
 456 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 457         [MLX5_EXPANSION_ROOT] = {
 458                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 459                                                   MLX5_EXPANSION_IPV4,
 460                                                   MLX5_EXPANSION_IPV6),
 461                 .type = RTE_FLOW_ITEM_TYPE_END,
 462         },
 463         [MLX5_EXPANSION_ROOT_OUTER] = {
 464                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 465                                                   MLX5_EXPANSION_OUTER_IPV4,
 466                                                   MLX5_EXPANSION_OUTER_IPV6),
 467                 .type = RTE_FLOW_ITEM_TYPE_END,
 468         },
 469         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 470                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 471                 .type = RTE_FLOW_ITEM_TYPE_END,
 472         },
 473         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 474                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 475                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 476                 .type = RTE_FLOW_ITEM_TYPE_END,
 477         },
 478         [MLX5_EXPANSION_OUTER_ETH] = {
 479                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 480                                                   MLX5_EXPANSION_OUTER_IPV6,
 481                                                   MLX5_EXPANSION_MPLS),
 482                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 483                 .rss_types = 0,
 484         },
 485         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 486                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 487                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 488                 .rss_types = 0,
 489         },
 490         [MLX5_EXPANSION_OUTER_VLAN] = {
 491                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 492                                                   MLX5_EXPANSION_OUTER_IPV6),
 493                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 494         },
 495         [MLX5_EXPANSION_OUTER_IPV4] = {
 496                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 497                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 498                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 499                          MLX5_EXPANSION_GRE,
 500                          MLX5_EXPANSION_NVGRE,
 501                          MLX5_EXPANSION_IPV4,
 502                          MLX5_EXPANSION_IPV6),
 503                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 504                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 505                         ETH_RSS_NONFRAG_IPV4_OTHER,
 506         },
 507         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 508                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 509                                                   MLX5_EXPANSION_VXLAN_GPE),
 510                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 511                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 512         },
 513         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 514                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 515                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 516         },
 517         [MLX5_EXPANSION_OUTER_IPV6] = {
 518                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 519                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 520                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 521                          MLX5_EXPANSION_IPV4,
 522                          MLX5_EXPANSION_IPV6,
 523                          MLX5_EXPANSION_GRE,
 524                          MLX5_EXPANSION_NVGRE),
 525                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 526                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 527                         ETH_RSS_NONFRAG_IPV6_OTHER,
 528         },
 529         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 530                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 531                                                   MLX5_EXPANSION_VXLAN_GPE),
 532                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 533                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 534         },
 535         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 536                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 537                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 538         },
 539         [MLX5_EXPANSION_VXLAN] = {
 540                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 541                                                   MLX5_EXPANSION_IPV4,
 542                                                   MLX5_EXPANSION_IPV6),
 543                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 544         },
 545         [MLX5_EXPANSION_VXLAN_GPE] = {
 546                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 547                                                   MLX5_EXPANSION_IPV4,
 548                                                   MLX5_EXPANSION_IPV6),
 549                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 550         },
 551         [MLX5_EXPANSION_GRE] = {
 552                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 553                                                   MLX5_EXPANSION_IPV6,
 554                                                   MLX5_EXPANSION_GRE_KEY),
 555                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 556         },
 557         [MLX5_EXPANSION_GRE_KEY] = {
 558                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 559                                                   MLX5_EXPANSION_IPV6),
 560                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 561                 .optional = 1,
 562         },
 563         [MLX5_EXPANSION_NVGRE] = {
 564                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 565                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 566         },
 567         [MLX5_EXPANSION_MPLS] = {
 568                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 569                                                   MLX5_EXPANSION_IPV6),
 570                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 571         },
 572         [MLX5_EXPANSION_ETH] = {
 573                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 574                                                   MLX5_EXPANSION_IPV6),
 575                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 576         },
 577         [MLX5_EXPANSION_ETH_VLAN] = {
 578                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 579                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 580         },
 581         [MLX5_EXPANSION_VLAN] = {
 582                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 583                                                   MLX5_EXPANSION_IPV6),
 584                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 585         },
 586         [MLX5_EXPANSION_IPV4] = {
 587                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 588                                                   MLX5_EXPANSION_IPV4_TCP),
 589                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 590                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 591                         ETH_RSS_NONFRAG_IPV4_OTHER,
 592         },
 593         [MLX5_EXPANSION_IPV4_UDP] = {
 594                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 595                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 596         },
 597         [MLX5_EXPANSION_IPV4_TCP] = {
 598                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 599                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 600         },
 601         [MLX5_EXPANSION_IPV6] = {
 602                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 603                                                   MLX5_EXPANSION_IPV6_TCP),
 604                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 605                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 606                         ETH_RSS_NONFRAG_IPV6_OTHER,
 607         },
 608         [MLX5_EXPANSION_IPV6_UDP] = {
 609                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 610                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 611         },
 612         [MLX5_EXPANSION_IPV6_TCP] = {
 613                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 614                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 615         },
 616 };
 617
 618 static struct rte_flow_action_handle *
 619 mlx5_action_handle_create(struct rte_eth_dev *dev,
 620                           const struct rte_flow_indir_action_conf *conf,
 621                           const struct rte_flow_action *action,
 622                           struct rte_flow_error *error);
 623 static int mlx5_action_handle_destroy
 624                                 (struct rte_eth_dev *dev,
 625                                  struct rte_flow_action_handle *handle,
 626                                  struct rte_flow_error *error);
 627 static int mlx5_action_handle_update
 628                                 (struct rte_eth_dev *dev,
 629                                  struct rte_flow_action_handle *handle,
 630                                  const void *update,
 631                                  struct rte_flow_error *error);
 632 static int mlx5_action_handle_query
 633                                 (struct rte_eth_dev *dev,
 634                                  const struct rte_flow_action_handle *handle,
 635                                  void *data,
 636                                  struct rte_flow_error *error);
 637 static int
 638 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 639                     struct rte_flow_tunnel *app_tunnel,
 640                     struct rte_flow_action **actions,
 641                     uint32_t *num_of_actions,
 642                     struct rte_flow_error *error);
 643 static int
 644 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 645                        struct rte_flow_tunnel *app_tunnel,
 646                        struct rte_flow_item **items,
 647                        uint32_t *num_of_items,
 648                        struct rte_flow_error *error);
 649 static int
 650 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 651                               struct rte_flow_item *pmd_items,
 652                               uint32_t num_items, struct rte_flow_error *err);
 653 static int
 654 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 655                                 struct rte_flow_action *pmd_actions,
 656                                 uint32_t num_actions,
 657                                 struct rte_flow_error *err);
 658 static int
 659 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 660                                   struct rte_mbuf *m,
 661                                   struct rte_flow_restore_info *info,
 662                                   struct rte_flow_error *err);
 663
 664 static const struct rte_flow_ops mlx5_flow_ops = {
 665         .validate = mlx5_flow_validate,
 666         .create = mlx5_flow_create,
 667         .destroy = mlx5_flow_destroy,
 668         .flush = mlx5_flow_flush,
 669         .isolate = mlx5_flow_isolate,
 670         .query = mlx5_flow_query,
 671         .dev_dump = mlx5_flow_dev_dump,
 672         .get_aged_flows = mlx5_flow_get_aged_flows,
 673         .action_handle_create = mlx5_action_handle_create,
 674         .action_handle_destroy = mlx5_action_handle_destroy,
 675         .action_handle_update = mlx5_action_handle_update,
 676         .action_handle_query = mlx5_action_handle_query,
 677         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 678         .tunnel_match = mlx5_flow_tunnel_match,
 679         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 680         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 681         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 682 };
 683
 684 /* Tunnel information. */
 685 struct mlx5_flow_tunnel_info {
 686         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 687         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 688 };
 689
 690 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 691         {
 692                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 693                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 694         },
 695         {
 696                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 697                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 698         },
 699         {
 700                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 701                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 702         },
 703         {
 704                 .tunnel = MLX5_FLOW_LAYER_GRE,
 705                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 706         },
 707         {
 708                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 709                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 710         },
 711         {
 712                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 713                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 714         },
 715         {
 716                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 717                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 718         },
 719         {
 720                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 721                 .ptype = RTE_PTYPE_TUNNEL_IP,
 722         },
 723         {
 724                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 725                 .ptype = RTE_PTYPE_TUNNEL_IP,
 726         },
 727         {
 728                 .tunnel = MLX5_FLOW_LAYER_GTP,
 729                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 730         },
 731 };
 732
 733
 734
 735 /**
 736  * Translate tag ID to register.
 737  *
 738  * @param[in] dev
 739  *   Pointer to the Ethernet device structure.
 740  * @param[in] feature
 741  *   The feature that request the register.
 742  * @param[in] id
 743  *   The request register ID.
 744  * @param[out] error
 745  *   Error description in case of any.
 746  *
 747  * @return
 748  *   The request register on success, a negative errno
 749  *   value otherwise and rte_errno is set.
 750  */
 751 int
 752 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 753                      enum mlx5_feature_name feature,
 754                      uint32_t id,
 755                      struct rte_flow_error *error)
 756 {
 757         struct mlx5_priv *priv = dev->data->dev_private;
 758         struct mlx5_dev_config *config = &priv->config;
 759         enum modify_reg start_reg;
 760         bool skip_mtr_reg = false;
 761
 762         switch (feature) {
 763         case MLX5_HAIRPIN_RX:
 764                 return REG_B;
 765         case MLX5_HAIRPIN_TX:
 766                 return REG_A;
 767         case MLX5_METADATA_RX:
 768                 switch (config->dv_xmeta_en) {
 769                 case MLX5_XMETA_MODE_LEGACY:
 770                         return REG_B;
 771                 case MLX5_XMETA_MODE_META16:
 772                         return REG_C_0;
 773                 case MLX5_XMETA_MODE_META32:
 774                         return REG_C_1;
 775                 }
 776                 break;
 777         case MLX5_METADATA_TX:
 778                 return REG_A;
 779         case MLX5_METADATA_FDB:
 780                 switch (config->dv_xmeta_en) {
 781                 case MLX5_XMETA_MODE_LEGACY:
 782                         return REG_NON;
 783                 case MLX5_XMETA_MODE_META16:
 784                         return REG_C_0;
 785                 case MLX5_XMETA_MODE_META32:
 786                         return REG_C_1;
 787                 }
 788                 break;
 789         case MLX5_FLOW_MARK:
 790                 switch (config->dv_xmeta_en) {
 791                 case MLX5_XMETA_MODE_LEGACY:
 792                         return REG_NON;
 793                 case MLX5_XMETA_MODE_META16:
 794                         return REG_C_1;
 795                 case MLX5_XMETA_MODE_META32:
 796                         return REG_C_0;
 797                 }
 798                 break;
 799         case MLX5_MTR_ID:
 800                 /*
 801                  * If meter color and meter id share one register, flow match
 802                  * should use the meter color register for match.
 803                  */
 804                 if (priv->mtr_reg_share)
 805                         return priv->mtr_color_reg;
 806                 else
 807                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 808                                REG_C_3;
 809         case MLX5_MTR_COLOR:
 810         case MLX5_ASO_FLOW_HIT:
 811         case MLX5_ASO_CONNTRACK:
 812                 /* All features use the same REG_C. */
 813                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 814                 return priv->mtr_color_reg;
 815         case MLX5_COPY_MARK:
 816                 /*
 817                  * Metadata COPY_MARK register using is in meter suffix sub
 818                  * flow while with meter. It's safe to share the same register.
 819                  */
 820                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 821         case MLX5_APP_TAG:
 822                 /*
 823                  * If meter is enable, it will engage the register for color
 824                  * match and flow match. If meter color match is not using the
 825                  * REG_C_2, need to skip the REG_C_x be used by meter color
 826                  * match.
 827                  * If meter is disable, free to use all available registers.
 828                  */
 829                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 830                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 831                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 832                 if (id > (uint32_t)(REG_C_7 - start_reg))
 833                         return rte_flow_error_set(error, EINVAL,
 834                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 835                                                   NULL, "invalid tag id");
 836                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 837                         return rte_flow_error_set(error, ENOTSUP,
 838                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 839                                                   NULL, "unsupported tag id");
 840                 /*
 841                  * This case means meter is using the REG_C_x great than 2.
 842                  * Take care not to conflict with meter color REG_C_x.
 843                  * If the available index REG_C_y >= REG_C_x, skip the
 844                  * color register.
 845                  */
 846                 if (skip_mtr_reg && config->flow_mreg_c
 847                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 848                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 849                                 return rte_flow_error_set(error, EINVAL,
 850                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 851                                                         NULL, "invalid tag id");
 852                         if (config->flow_mreg_c
 853                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 854                                 return config->flow_mreg_c
 855                                                [id + 1 + start_reg - REG_C_0];
 856                         return rte_flow_error_set(error, ENOTSUP,
 857                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 858                                                   NULL, "unsupported tag id");
 859                 }
 860                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 861         }
 862         MLX5_ASSERT(false);
 863         return rte_flow_error_set(error, EINVAL,
 864                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 865                                   NULL, "invalid feature name");
 866 }
 867
 868 /**
 869  * Check extensive flow metadata register support.
 870  *
 871  * @param dev
 872  *   Pointer to rte_eth_dev structure.
 873  *
 874  * @return
 875  *   True if device supports extensive flow metadata register, otherwise false.
 876  */
 877 bool
 878 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 879 {
 880         struct mlx5_priv *priv = dev->data->dev_private;
 881         struct mlx5_dev_config *config = &priv->config;
 882
 883         /*
 884          * Having available reg_c can be regarded inclusively as supporting
 885          * extensive flow metadata register, which could mean,
 886          * - metadata register copy action by modify header.
 887          * - 16 modify header actions is supported.
 888          * - reg_c's are preserved across different domain (FDB and NIC) on
 889          *   packet loopback by flow lookup miss.
 890          */
 891         return config->flow_mreg_c[2] != REG_NON;
 892 }
 893
 894 /**
 895  * Get the lowest priority.
 896  *
 897  * @param[in] dev
 898  *   Pointer to the Ethernet device structure.
 899  * @param[in] attributes
 900  *   Pointer to device flow rule attributes.
 901  *
 902  * @return
 903  *   The value of lowest priority of flow.
 904  */
 905 uint32_t
 906 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 907                           const struct rte_flow_attr *attr)
 908 {
 909         struct mlx5_priv *priv = dev->data->dev_private;
 910
 911         if (!attr->group && !attr->transfer)
 912                 return priv->config.flow_prio - 2;
 913         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 914 }
 915
 916 /**
 917  * Calculate matcher priority of the flow.
 918  *
 919  * @param[in] dev
 920  *   Pointer to the Ethernet device structure.
 921  * @param[in] attr
 922  *   Pointer to device flow rule attributes.
 923  * @param[in] subpriority
 924  *   The priority based on the items.
 925  * @return
 926  *   The matcher priority of the flow.
 927  */
 928 uint16_t
 929 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 930                           const struct rte_flow_attr *attr,
 931                           uint32_t subpriority)
 932 {
 933         uint16_t priority = (uint16_t)attr->priority;
 934         struct mlx5_priv *priv = dev->data->dev_private;
 935
 936         if (!attr->group && !attr->transfer) {
 937                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 938                         priority = priv->config.flow_prio - 1;
 939                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 940         }
 941         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 942                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 943         return priority * 3 + subpriority;
 944 }
 945
 946 /**
 947  * Verify the @p item specifications (spec, last, mask) are compatible with the
 948  * NIC capabilities.
 949  *
 950  * @param[in] item
 951  *   Item specification.
 952  * @param[in] mask
 953  *   @p item->mask or flow default bit-masks.
 954  * @param[in] nic_mask
 955  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 956  * @param[in] size
 957  *   Bit-masks size in bytes.
 958  * @param[in] range_accepted
 959  *   True if range of values is accepted for specific fields, false otherwise.
 960  * @param[out] error
 961  *   Pointer to error structure.
 962  *
 963  * @return
 964  *   0 on success, a negative errno value otherwise and rte_errno is set.
 965  */
 966 int
 967 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 968                           const uint8_t *mask,
 969                           const uint8_t *nic_mask,
 970                           unsigned int size,
 971                           bool range_accepted,
 972                           struct rte_flow_error *error)
 973 {
 974         unsigned int i;
 975
 976         MLX5_ASSERT(nic_mask);
 977         for (i = 0; i < size; ++i)
 978                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 979                         return rte_flow_error_set(error, ENOTSUP,
 980                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 981                                                   item,
 982                                                   "mask enables non supported"
 983                                                   " bits");
 984         if (!item->spec && (item->mask || item->last))
 985                 return rte_flow_error_set(error, EINVAL,
 986                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 987                                           "mask/last without a spec is not"
 988                                           " supported");
 989         if (item->spec && item->last && !range_accepted) {
 990                 uint8_t spec[size];
 991                 uint8_t last[size];
 992                 unsigned int i;
 993                 int ret;
 994
 995                 for (i = 0; i < size; ++i) {
 996                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 997                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 998                 }
 999                 ret = memcmp(spec, last, size);
1000                 if (ret != 0)
1001                         return rte_flow_error_set(error, EINVAL,
1002                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1003                                                   item,
1004                                                   "range is not valid");
1005         }
1006         return 0;
1007 }
1008
1009 /**
1010  * Adjust the hash fields according to the @p flow information.
1011  *
1012  * @param[in] dev_flow.
1013  *   Pointer to the mlx5_flow.
1014  * @param[in] tunnel
1015  *   1 when the hash field is for a tunnel item.
1016  * @param[in] layer_types
1017  *   ETH_RSS_* types.
1018  * @param[in] hash_fields
1019  *   Item hash fields.
1020  *
1021  * @return
1022  *   The hash fields that should be used.
1023  */
1024 uint64_t
1025 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1026                             int tunnel __rte_unused, uint64_t layer_types,
1027                             uint64_t hash_fields)
1028 {
1029 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1030         int rss_request_inner = rss_desc->level >= 2;
1031
1032         /* Check RSS hash level for tunnel. */
1033         if (tunnel && rss_request_inner)
1034                 hash_fields |= IBV_RX_HASH_INNER;
1035         else if (tunnel || rss_request_inner)
1036                 return 0;
1037 #endif
1038         /* Check if requested layer matches RSS hash fields. */
1039         if (!(rss_desc->types & layer_types))
1040                 return 0;
1041         return hash_fields;
1042 }
1043
1044 /**
1045  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1046  * if several tunnel rules are used on this queue, the tunnel ptype will be
1047  * cleared.
1048  *
1049  * @param rxq_ctrl
1050  *   Rx queue to update.
1051  */
1052 static void
1053 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1054 {
1055         unsigned int i;
1056         uint32_t tunnel_ptype = 0;
1057
1058         /* Look up for the ptype to use. */
1059         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1060                 if (!rxq_ctrl->flow_tunnels_n[i])
1061                         continue;
1062                 if (!tunnel_ptype) {
1063                         tunnel_ptype = tunnels_info[i].ptype;
1064                 } else {
1065                         tunnel_ptype = 0;
1066                         break;
1067                 }
1068         }
1069         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1070 }
1071
1072 /**
1073  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1074  * flow.
1075  *
1076  * @param[in] dev
1077  *   Pointer to the Ethernet device structure.
1078  * @param[in] dev_handle
1079  *   Pointer to device flow handle structure.
1080  */
1081 void
1082 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1083                        struct mlx5_flow_handle *dev_handle)
1084 {
1085         struct mlx5_priv *priv = dev->data->dev_private;
1086         const int mark = dev_handle->mark;
1087         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1088         struct mlx5_ind_table_obj *ind_tbl = NULL;
1089         unsigned int i;
1090
1091         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1092                 struct mlx5_hrxq *hrxq;
1093
1094                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1095                               dev_handle->rix_hrxq);
1096                 if (hrxq)
1097                         ind_tbl = hrxq->ind_table;
1098         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1099                 struct mlx5_shared_action_rss *shared_rss;
1100
1101                 shared_rss = mlx5_ipool_get
1102                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1103                          dev_handle->rix_srss);
1104                 if (shared_rss)
1105                         ind_tbl = shared_rss->ind_tbl;
1106         }
1107         if (!ind_tbl)
1108                 return;
1109         for (i = 0; i != ind_tbl->queues_n; ++i) {
1110                 int idx = ind_tbl->queues[i];
1111                 struct mlx5_rxq_ctrl *rxq_ctrl =
1112                         container_of((*priv->rxqs)[idx],
1113                                      struct mlx5_rxq_ctrl, rxq);
1114
1115                 /*
1116                  * To support metadata register copy on Tx loopback,
1117                  * this must be always enabled (metadata may arive
1118                  * from other port - not from local flows only.
1119                  */
1120                 if (priv->config.dv_flow_en &&
1121                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1122                     mlx5_flow_ext_mreg_supported(dev)) {
1123                         rxq_ctrl->rxq.mark = 1;
1124                         rxq_ctrl->flow_mark_n = 1;
1125                 } else if (mark) {
1126                         rxq_ctrl->rxq.mark = 1;
1127                         rxq_ctrl->flow_mark_n++;
1128                 }
1129                 if (tunnel) {
1130                         unsigned int j;
1131
1132                         /* Increase the counter matching the flow. */
1133                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1134                                 if ((tunnels_info[j].tunnel &
1135                                      dev_handle->layers) ==
1136                                     tunnels_info[j].tunnel) {
1137                                         rxq_ctrl->flow_tunnels_n[j]++;
1138                                         break;
1139                                 }
1140                         }
1141                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1142                 }
1143         }
1144 }
1145
1146 /**
1147  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1148  *
1149  * @param[in] dev
1150  *   Pointer to the Ethernet device structure.
1151  * @param[in] flow
1152  *   Pointer to flow structure.
1153  */
1154 static void
1155 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1156 {
1157         struct mlx5_priv *priv = dev->data->dev_private;
1158         uint32_t handle_idx;
1159         struct mlx5_flow_handle *dev_handle;
1160
1161         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1162                        handle_idx, dev_handle, next)
1163                 flow_drv_rxq_flags_set(dev, dev_handle);
1164 }
1165
1166 /**
1167  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1168  * device flow if no other flow uses it with the same kind of request.
1169  *
1170  * @param dev
1171  *   Pointer to Ethernet device.
1172  * @param[in] dev_handle
1173  *   Pointer to the device flow handle structure.
1174  */
1175 static void
1176 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1177                         struct mlx5_flow_handle *dev_handle)
1178 {
1179         struct mlx5_priv *priv = dev->data->dev_private;
1180         const int mark = dev_handle->mark;
1181         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1182         struct mlx5_ind_table_obj *ind_tbl = NULL;
1183         unsigned int i;
1184
1185         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1186                 struct mlx5_hrxq *hrxq;
1187
1188                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1189                               dev_handle->rix_hrxq);
1190                 if (hrxq)
1191                         ind_tbl = hrxq->ind_table;
1192         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1193                 struct mlx5_shared_action_rss *shared_rss;
1194
1195                 shared_rss = mlx5_ipool_get
1196                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1197                          dev_handle->rix_srss);
1198                 if (shared_rss)
1199                         ind_tbl = shared_rss->ind_tbl;
1200         }
1201         if (!ind_tbl)
1202                 return;
1203         MLX5_ASSERT(dev->data->dev_started);
1204         for (i = 0; i != ind_tbl->queues_n; ++i) {
1205                 int idx = ind_tbl->queues[i];
1206                 struct mlx5_rxq_ctrl *rxq_ctrl =
1207                         container_of((*priv->rxqs)[idx],
1208                                      struct mlx5_rxq_ctrl, rxq);
1209
1210                 if (priv->config.dv_flow_en &&
1211                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1212                     mlx5_flow_ext_mreg_supported(dev)) {
1213                         rxq_ctrl->rxq.mark = 1;
1214                         rxq_ctrl->flow_mark_n = 1;
1215                 } else if (mark) {
1216                         rxq_ctrl->flow_mark_n--;
1217                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1218                 }
1219                 if (tunnel) {
1220                         unsigned int j;
1221
1222                         /* Decrease the counter matching the flow. */
1223                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1224                                 if ((tunnels_info[j].tunnel &
1225                                      dev_handle->layers) ==
1226                                     tunnels_info[j].tunnel) {
1227                                         rxq_ctrl->flow_tunnels_n[j]--;
1228                                         break;
1229                                 }
1230                         }
1231                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1232                 }
1233         }
1234 }
1235
1236 /**
1237  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1238  * @p flow if no other flow uses it with the same kind of request.
1239  *
1240  * @param dev
1241  *   Pointer to Ethernet device.
1242  * @param[in] flow
1243  *   Pointer to the flow.
1244  */
1245 static void
1246 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1247 {
1248         struct mlx5_priv *priv = dev->data->dev_private;
1249         uint32_t handle_idx;
1250         struct mlx5_flow_handle *dev_handle;
1251
1252         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1253                        handle_idx, dev_handle, next)
1254                 flow_drv_rxq_flags_trim(dev, dev_handle);
1255 }
1256
1257 /**
1258  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1259  *
1260  * @param dev
1261  *   Pointer to Ethernet device.
1262  */
1263 static void
1264 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1265 {
1266         struct mlx5_priv *priv = dev->data->dev_private;
1267         unsigned int i;
1268
1269         for (i = 0; i != priv->rxqs_n; ++i) {
1270                 struct mlx5_rxq_ctrl *rxq_ctrl;
1271                 unsigned int j;
1272
1273                 if (!(*priv->rxqs)[i])
1274                         continue;
1275                 rxq_ctrl = container_of((*priv->rxqs)[i],
1276                                         struct mlx5_rxq_ctrl, rxq);
1277                 rxq_ctrl->flow_mark_n = 0;
1278                 rxq_ctrl->rxq.mark = 0;
1279                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1280                         rxq_ctrl->flow_tunnels_n[j] = 0;
1281                 rxq_ctrl->rxq.tunnel = 0;
1282         }
1283 }
1284
1285 /**
1286  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1287  *
1288  * @param[in] dev
1289  *   Pointer to the Ethernet device structure.
1290  */
1291 void
1292 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1293 {
1294         struct mlx5_priv *priv = dev->data->dev_private;
1295         struct mlx5_rxq_data *data;
1296         unsigned int i;
1297
1298         for (i = 0; i != priv->rxqs_n; ++i) {
1299                 if (!(*priv->rxqs)[i])
1300                         continue;
1301                 data = (*priv->rxqs)[i];
1302                 if (!rte_flow_dynf_metadata_avail()) {
1303                         data->dynf_meta = 0;
1304                         data->flow_meta_mask = 0;
1305                         data->flow_meta_offset = -1;
1306                         data->flow_meta_port_mask = 0;
1307                 } else {
1308                         data->dynf_meta = 1;
1309                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1310                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1311                         data->flow_meta_port_mask = (uint32_t)~0;
1312                         if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16)
1313                                 data->flow_meta_port_mask >>= 16;
1314                 }
1315         }
1316 }
1317
1318 /*
1319  * return a pointer to the desired action in the list of actions.
1320  *
1321  * @param[in] actions
1322  *   The list of actions to search the action in.
1323  * @param[in] action
1324  *   The action to find.
1325  *
1326  * @return
1327  *   Pointer to the action in the list, if found. NULL otherwise.
1328  */
1329 const struct rte_flow_action *
1330 mlx5_flow_find_action(const struct rte_flow_action *actions,
1331                       enum rte_flow_action_type action)
1332 {
1333         if (actions == NULL)
1334                 return NULL;
1335         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1336                 if (actions->type == action)
1337                         return actions;
1338         return NULL;
1339 }
1340
1341 /*
1342  * Validate the flag action.
1343  *
1344  * @param[in] action_flags
1345  *   Bit-fields that holds the actions detected until now.
1346  * @param[in] attr
1347  *   Attributes of flow that includes this action.
1348  * @param[out] error
1349  *   Pointer to error structure.
1350  *
1351  * @return
1352  *   0 on success, a negative errno value otherwise and rte_errno is set.
1353  */
1354 int
1355 mlx5_flow_validate_action_flag(uint64_t action_flags,
1356                                const struct rte_flow_attr *attr,
1357                                struct rte_flow_error *error)
1358 {
1359         if (action_flags & MLX5_FLOW_ACTION_MARK)
1360                 return rte_flow_error_set(error, EINVAL,
1361                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1362                                           "can't mark and flag in same flow");
1363         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1364                 return rte_flow_error_set(error, EINVAL,
1365                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1366                                           "can't have 2 flag"
1367                                           " actions in same flow");
1368         if (attr->egress)
1369                 return rte_flow_error_set(error, ENOTSUP,
1370                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1371                                           "flag action not supported for "
1372                                           "egress");
1373         return 0;
1374 }
1375
1376 /*
1377  * Validate the mark action.
1378  *
1379  * @param[in] action
1380  *   Pointer to the queue action.
1381  * @param[in] action_flags
1382  *   Bit-fields that holds the actions detected until now.
1383  * @param[in] attr
1384  *   Attributes of flow that includes this action.
1385  * @param[out] error
1386  *   Pointer to error structure.
1387  *
1388  * @return
1389  *   0 on success, a negative errno value otherwise and rte_errno is set.
1390  */
1391 int
1392 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1393                                uint64_t action_flags,
1394                                const struct rte_flow_attr *attr,
1395                                struct rte_flow_error *error)
1396 {
1397         const struct rte_flow_action_mark *mark = action->conf;
1398
1399         if (!mark)
1400                 return rte_flow_error_set(error, EINVAL,
1401                                           RTE_FLOW_ERROR_TYPE_ACTION,
1402                                           action,
1403                                           "configuration cannot be null");
1404         if (mark->id >= MLX5_FLOW_MARK_MAX)
1405                 return rte_flow_error_set(error, EINVAL,
1406                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1407                                           &mark->id,
1408                                           "mark id must in 0 <= id < "
1409                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1410         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1411                 return rte_flow_error_set(error, EINVAL,
1412                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1413                                           "can't flag and mark in same flow");
1414         if (action_flags & MLX5_FLOW_ACTION_MARK)
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1417                                           "can't have 2 mark actions in same"
1418                                           " flow");
1419         if (attr->egress)
1420                 return rte_flow_error_set(error, ENOTSUP,
1421                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1422                                           "mark action not supported for "
1423                                           "egress");
1424         return 0;
1425 }
1426
1427 /*
1428  * Validate the drop action.
1429  *
1430  * @param[in] action_flags
1431  *   Bit-fields that holds the actions detected until now.
1432  * @param[in] attr
1433  *   Attributes of flow that includes this action.
1434  * @param[out] error
1435  *   Pointer to error structure.
1436  *
1437  * @return
1438  *   0 on success, a negative errno value otherwise and rte_errno is set.
1439  */
1440 int
1441 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1442                                const struct rte_flow_attr *attr,
1443                                struct rte_flow_error *error)
1444 {
1445         if (attr->egress)
1446                 return rte_flow_error_set(error, ENOTSUP,
1447                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1448                                           "drop action not supported for "
1449                                           "egress");
1450         return 0;
1451 }
1452
1453 /*
1454  * Validate the queue action.
1455  *
1456  * @param[in] action
1457  *   Pointer to the queue action.
1458  * @param[in] action_flags
1459  *   Bit-fields that holds the actions detected until now.
1460  * @param[in] dev
1461  *   Pointer to the Ethernet device structure.
1462  * @param[in] attr
1463  *   Attributes of flow that includes this action.
1464  * @param[out] error
1465  *   Pointer to error structure.
1466  *
1467  * @return
1468  *   0 on success, a negative errno value otherwise and rte_errno is set.
1469  */
1470 int
1471 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1472                                 uint64_t action_flags,
1473                                 struct rte_eth_dev *dev,
1474                                 const struct rte_flow_attr *attr,
1475                                 struct rte_flow_error *error)
1476 {
1477         struct mlx5_priv *priv = dev->data->dev_private;
1478         const struct rte_flow_action_queue *queue = action->conf;
1479
1480         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1481                 return rte_flow_error_set(error, EINVAL,
1482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1483                                           "can't have 2 fate actions in"
1484                                           " same flow");
1485         if (!priv->rxqs_n)
1486                 return rte_flow_error_set(error, EINVAL,
1487                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1488                                           NULL, "No Rx queues configured");
1489         if (queue->index >= priv->rxqs_n)
1490                 return rte_flow_error_set(error, EINVAL,
1491                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1492                                           &queue->index,
1493                                           "queue index out of range");
1494         if (!(*priv->rxqs)[queue->index])
1495                 return rte_flow_error_set(error, EINVAL,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1497                                           &queue->index,
1498                                           "queue is not configured");
1499         if (attr->egress)
1500                 return rte_flow_error_set(error, ENOTSUP,
1501                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1502                                           "queue action not supported for "
1503                                           "egress");
1504         return 0;
1505 }
1506
1507 /*
1508  * Validate the rss action.
1509  *
1510  * @param[in] dev
1511  *   Pointer to the Ethernet device structure.
1512  * @param[in] action
1513  *   Pointer to the queue action.
1514  * @param[out] error
1515  *   Pointer to error structure.
1516  *
1517  * @return
1518  *   0 on success, a negative errno value otherwise and rte_errno is set.
1519  */
1520 int
1521 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1522                          const struct rte_flow_action *action,
1523                          struct rte_flow_error *error)
1524 {
1525         struct mlx5_priv *priv = dev->data->dev_private;
1526         const struct rte_flow_action_rss *rss = action->conf;
1527         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1528         unsigned int i;
1529
1530         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1531             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1532                 return rte_flow_error_set(error, ENOTSUP,
1533                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1534                                           &rss->func,
1535                                           "RSS hash function not supported");
1536 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1537         if (rss->level > 2)
1538 #else
1539         if (rss->level > 1)
1540 #endif
1541                 return rte_flow_error_set(error, ENOTSUP,
1542                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1543                                           &rss->level,
1544                                           "tunnel RSS is not supported");
1545         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1546         if (rss->key_len == 0 && rss->key != NULL)
1547                 return rte_flow_error_set(error, ENOTSUP,
1548                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1549                                           &rss->key_len,
1550                                           "RSS hash key length 0");
1551         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1552                 return rte_flow_error_set(error, ENOTSUP,
1553                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1554                                           &rss->key_len,
1555                                           "RSS hash key too small");
1556         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1557                 return rte_flow_error_set(error, ENOTSUP,
1558                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1559                                           &rss->key_len,
1560                                           "RSS hash key too large");
1561         if (rss->queue_num > priv->config.ind_table_max_size)
1562                 return rte_flow_error_set(error, ENOTSUP,
1563                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1564                                           &rss->queue_num,
1565                                           "number of queues too large");
1566         if (rss->types & MLX5_RSS_HF_MASK)
1567                 return rte_flow_error_set(error, ENOTSUP,
1568                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1569                                           &rss->types,
1570                                           "some RSS protocols are not"
1571                                           " supported");
1572         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1573             !(rss->types & ETH_RSS_IP))
1574                 return rte_flow_error_set(error, EINVAL,
1575                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1576                                           "L3 partial RSS requested but L3 RSS"
1577                                           " type not specified");
1578         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1579             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1580                 return rte_flow_error_set(error, EINVAL,
1581                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1582                                           "L4 partial RSS requested but L4 RSS"
1583                                           " type not specified");
1584         if (!priv->rxqs_n)
1585                 return rte_flow_error_set(error, EINVAL,
1586                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1587                                           NULL, "No Rx queues configured");
1588         if (!rss->queue_num)
1589                 return rte_flow_error_set(error, EINVAL,
1590                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1591                                           NULL, "No queues configured");
1592         for (i = 0; i != rss->queue_num; ++i) {
1593                 struct mlx5_rxq_ctrl *rxq_ctrl;
1594
1595                 if (rss->queue[i] >= priv->rxqs_n)
1596                         return rte_flow_error_set
1597                                 (error, EINVAL,
1598                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1599                                  &rss->queue[i], "queue index out of range");
1600                 if (!(*priv->rxqs)[rss->queue[i]])
1601                         return rte_flow_error_set
1602                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1603                                  &rss->queue[i], "queue is not configured");
1604                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1605                                         struct mlx5_rxq_ctrl, rxq);
1606                 if (i == 0)
1607                         rxq_type = rxq_ctrl->type;
1608                 if (rxq_type != rxq_ctrl->type)
1609                         return rte_flow_error_set
1610                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1611                                  &rss->queue[i],
1612                                  "combining hairpin and regular RSS queues is not supported");
1613         }
1614         return 0;
1615 }
1616
1617 /*
1618  * Validate the rss action.
1619  *
1620  * @param[in] action
1621  *   Pointer to the queue action.
1622  * @param[in] action_flags
1623  *   Bit-fields that holds the actions detected until now.
1624  * @param[in] dev
1625  *   Pointer to the Ethernet device structure.
1626  * @param[in] attr
1627  *   Attributes of flow that includes this action.
1628  * @param[in] item_flags
1629  *   Items that were detected.
1630  * @param[out] error
1631  *   Pointer to error structure.
1632  *
1633  * @return
1634  *   0 on success, a negative errno value otherwise and rte_errno is set.
1635  */
1636 int
1637 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1638                               uint64_t action_flags,
1639                               struct rte_eth_dev *dev,
1640                               const struct rte_flow_attr *attr,
1641                               uint64_t item_flags,
1642                               struct rte_flow_error *error)
1643 {
1644         const struct rte_flow_action_rss *rss = action->conf;
1645         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1646         int ret;
1647
1648         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1649                 return rte_flow_error_set(error, EINVAL,
1650                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1651                                           "can't have 2 fate actions"
1652                                           " in same flow");
1653         ret = mlx5_validate_action_rss(dev, action, error);
1654         if (ret)
1655                 return ret;
1656         if (attr->egress)
1657                 return rte_flow_error_set(error, ENOTSUP,
1658                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1659                                           "rss action not supported for "
1660                                           "egress");
1661         if (rss->level > 1 && !tunnel)
1662                 return rte_flow_error_set(error, EINVAL,
1663                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1664                                           "inner RSS is not supported for "
1665                                           "non-tunnel flows");
1666         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1667             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1668                 return rte_flow_error_set(error, EINVAL,
1669                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1670                                           "RSS on eCPRI is not supported now");
1671         }
1672         return 0;
1673 }
1674
1675 /*
1676  * Validate the default miss action.
1677  *
1678  * @param[in] action_flags
1679  *   Bit-fields that holds the actions detected until now.
1680  * @param[out] error
1681  *   Pointer to error structure.
1682  *
1683  * @return
1684  *   0 on success, a negative errno value otherwise and rte_errno is set.
1685  */
1686 int
1687 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1688                                 const struct rte_flow_attr *attr,
1689                                 struct rte_flow_error *error)
1690 {
1691         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1692                 return rte_flow_error_set(error, EINVAL,
1693                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1694                                           "can't have 2 fate actions in"
1695                                           " same flow");
1696         if (attr->egress)
1697                 return rte_flow_error_set(error, ENOTSUP,
1698                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1699                                           "default miss action not supported "
1700                                           "for egress");
1701         if (attr->group)
1702                 return rte_flow_error_set(error, ENOTSUP,
1703                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1704                                           "only group 0 is supported");
1705         if (attr->transfer)
1706                 return rte_flow_error_set(error, ENOTSUP,
1707                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1708                                           NULL, "transfer is not supported");
1709         return 0;
1710 }
1711
1712 /*
1713  * Validate the count action.
1714  *
1715  * @param[in] dev
1716  *   Pointer to the Ethernet device structure.
1717  * @param[in] attr
1718  *   Attributes of flow that includes this action.
1719  * @param[out] error
1720  *   Pointer to error structure.
1721  *
1722  * @return
1723  *   0 on success, a negative errno value otherwise and rte_errno is set.
1724  */
1725 int
1726 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1727                                 const struct rte_flow_attr *attr,
1728                                 struct rte_flow_error *error)
1729 {
1730         if (attr->egress)
1731                 return rte_flow_error_set(error, ENOTSUP,
1732                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1733                                           "count action not supported for "
1734                                           "egress");
1735         return 0;
1736 }
1737
1738 /*
1739  * Validate the ASO CT action.
1740  *
1741  * @param[in] dev
1742  *   Pointer to the Ethernet device structure.
1743  * @param[in] conntrack
1744  *   Pointer to the CT action profile.
1745  * @param[out] error
1746  *   Pointer to error structure.
1747  *
1748  * @return
1749  *   0 on success, a negative errno value otherwise and rte_errno is set.
1750  */
1751 int
1752 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1753                         const struct rte_flow_action_conntrack *conntrack,
1754                         struct rte_flow_error *error)
1755 {
1756         RTE_SET_USED(dev);
1757
1758         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1759                 return rte_flow_error_set(error, EINVAL,
1760                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1761                                           "Invalid CT state");
1762         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1763                 return rte_flow_error_set(error, EINVAL,
1764                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1765                                           "Invalid last TCP packet flag");
1766         return 0;
1767 }
1768
1769 /**
1770  * Verify the @p attributes will be correctly understood by the NIC and store
1771  * them in the @p flow if everything is correct.
1772  *
1773  * @param[in] dev
1774  *   Pointer to the Ethernet device structure.
1775  * @param[in] attributes
1776  *   Pointer to flow attributes
1777  * @param[out] error
1778  *   Pointer to error structure.
1779  *
1780  * @return
1781  *   0 on success, a negative errno value otherwise and rte_errno is set.
1782  */
1783 int
1784 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1785                               const struct rte_flow_attr *attributes,
1786                               struct rte_flow_error *error)
1787 {
1788         struct mlx5_priv *priv = dev->data->dev_private;
1789         uint32_t priority_max = priv->config.flow_prio - 1;
1790
1791         if (attributes->group)
1792                 return rte_flow_error_set(error, ENOTSUP,
1793                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1794                                           NULL, "groups is not supported");
1795         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1796             attributes->priority >= priority_max)
1797                 return rte_flow_error_set(error, ENOTSUP,
1798                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1799                                           NULL, "priority out of range");
1800         if (attributes->egress)
1801                 return rte_flow_error_set(error, ENOTSUP,
1802                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1803                                           "egress is not supported");
1804         if (attributes->transfer && !priv->config.dv_esw_en)
1805                 return rte_flow_error_set(error, ENOTSUP,
1806                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1807                                           NULL, "transfer is not supported");
1808         if (!attributes->ingress)
1809                 return rte_flow_error_set(error, EINVAL,
1810                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1811                                           NULL,
1812                                           "ingress attribute is mandatory");
1813         return 0;
1814 }
1815
1816 /**
1817  * Validate ICMP6 item.
1818  *
1819  * @param[in] item
1820  *   Item specification.
1821  * @param[in] item_flags
1822  *   Bit-fields that holds the items detected until now.
1823  * @param[in] ext_vlan_sup
1824  *   Whether extended VLAN features are supported or not.
1825  * @param[out] error
1826  *   Pointer to error structure.
1827  *
1828  * @return
1829  *   0 on success, a negative errno value otherwise and rte_errno is set.
1830  */
1831 int
1832 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1833                                uint64_t item_flags,
1834                                uint8_t target_protocol,
1835                                struct rte_flow_error *error)
1836 {
1837         const struct rte_flow_item_icmp6 *mask = item->mask;
1838         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1839         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1840                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1841         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1842                                       MLX5_FLOW_LAYER_OUTER_L4;
1843         int ret;
1844
1845         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1846                 return rte_flow_error_set(error, EINVAL,
1847                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1848                                           "protocol filtering not compatible"
1849                                           " with ICMP6 layer");
1850         if (!(item_flags & l3m))
1851                 return rte_flow_error_set(error, EINVAL,
1852                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1853                                           "IPv6 is mandatory to filter on"
1854                                           " ICMP6");
1855         if (item_flags & l4m)
1856                 return rte_flow_error_set(error, EINVAL,
1857                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1858                                           "multiple L4 layers not supported");
1859         if (!mask)
1860                 mask = &rte_flow_item_icmp6_mask;
1861         ret = mlx5_flow_item_acceptable
1862                 (item, (const uint8_t *)mask,
1863                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1864                  sizeof(struct rte_flow_item_icmp6),
1865                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1866         if (ret < 0)
1867                 return ret;
1868         return 0;
1869 }
1870
1871 /**
1872  * Validate ICMP item.
1873  *
1874  * @param[in] item
1875  *   Item specification.
1876  * @param[in] item_flags
1877  *   Bit-fields that holds the items detected until now.
1878  * @param[out] error
1879  *   Pointer to error structure.
1880  *
1881  * @return
1882  *   0 on success, a negative errno value otherwise and rte_errno is set.
1883  */
1884 int
1885 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1886                              uint64_t item_flags,
1887                              uint8_t target_protocol,
1888                              struct rte_flow_error *error)
1889 {
1890         const struct rte_flow_item_icmp *mask = item->mask;
1891         const struct rte_flow_item_icmp nic_mask = {
1892                 .hdr.icmp_type = 0xff,
1893                 .hdr.icmp_code = 0xff,
1894                 .hdr.icmp_ident = RTE_BE16(0xffff),
1895                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1896         };
1897         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1898         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1899                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1900         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1901                                       MLX5_FLOW_LAYER_OUTER_L4;
1902         int ret;
1903
1904         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1905                 return rte_flow_error_set(error, EINVAL,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1907                                           "protocol filtering not compatible"
1908                                           " with ICMP layer");
1909         if (!(item_flags & l3m))
1910                 return rte_flow_error_set(error, EINVAL,
1911                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1912                                           "IPv4 is mandatory to filter"
1913                                           " on ICMP");
1914         if (item_flags & l4m)
1915                 return rte_flow_error_set(error, EINVAL,
1916                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1917                                           "multiple L4 layers not supported");
1918         if (!mask)
1919                 mask = &nic_mask;
1920         ret = mlx5_flow_item_acceptable
1921                 (item, (const uint8_t *)mask,
1922                  (const uint8_t *)&nic_mask,
1923                  sizeof(struct rte_flow_item_icmp),
1924                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1925         if (ret < 0)
1926                 return ret;
1927         return 0;
1928 }
1929
1930 /**
1931  * Validate Ethernet item.
1932  *
1933  * @param[in] item
1934  *   Item specification.
1935  * @param[in] item_flags
1936  *   Bit-fields that holds the items detected until now.
1937  * @param[out] error
1938  *   Pointer to error structure.
1939  *
1940  * @return
1941  *   0 on success, a negative errno value otherwise and rte_errno is set.
1942  */
1943 int
1944 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1945                             uint64_t item_flags, bool ext_vlan_sup,
1946                             struct rte_flow_error *error)
1947 {
1948         const struct rte_flow_item_eth *mask = item->mask;
1949         const struct rte_flow_item_eth nic_mask = {
1950                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1951                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1952                 .type = RTE_BE16(0xffff),
1953                 .has_vlan = ext_vlan_sup ? 1 : 0,
1954         };
1955         int ret;
1956         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1957         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1958                                        MLX5_FLOW_LAYER_OUTER_L2;
1959
1960         if (item_flags & ethm)
1961                 return rte_flow_error_set(error, ENOTSUP,
1962                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1963                                           "multiple L2 layers not supported");
1964         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1965             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1968                                           "L2 layer should not follow "
1969                                           "L3 layers");
1970         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1971             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1972                 return rte_flow_error_set(error, EINVAL,
1973                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1974                                           "L2 layer should not follow VLAN");
1975         if (!mask)
1976                 mask = &rte_flow_item_eth_mask;
1977         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1978                                         (const uint8_t *)&nic_mask,
1979                                         sizeof(struct rte_flow_item_eth),
1980                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1981         return ret;
1982 }
1983
1984 /**
1985  * Validate VLAN item.
1986  *
1987  * @param[in] item
1988  *   Item specification.
1989  * @param[in] item_flags
1990  *   Bit-fields that holds the items detected until now.
1991  * @param[in] dev
1992  *   Ethernet device flow is being created on.
1993  * @param[out] error
1994  *   Pointer to error structure.
1995  *
1996  * @return
1997  *   0 on success, a negative errno value otherwise and rte_errno is set.
1998  */
1999 int
2000 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2001                              uint64_t item_flags,
2002                              struct rte_eth_dev *dev,
2003                              struct rte_flow_error *error)
2004 {
2005         const struct rte_flow_item_vlan *spec = item->spec;
2006         const struct rte_flow_item_vlan *mask = item->mask;
2007         const struct rte_flow_item_vlan nic_mask = {
2008                 .tci = RTE_BE16(UINT16_MAX),
2009                 .inner_type = RTE_BE16(UINT16_MAX),
2010         };
2011         uint16_t vlan_tag = 0;
2012         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2013         int ret;
2014         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2015                                         MLX5_FLOW_LAYER_INNER_L4) :
2016                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2017                                         MLX5_FLOW_LAYER_OUTER_L4);
2018         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2019                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2020
2021         if (item_flags & vlanm)
2022                 return rte_flow_error_set(error, EINVAL,
2023                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2024                                           "multiple VLAN layers not supported");
2025         else if ((item_flags & l34m) != 0)
2026                 return rte_flow_error_set(error, EINVAL,
2027                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2028                                           "VLAN cannot follow L3/L4 layer");
2029         if (!mask)
2030                 mask = &rte_flow_item_vlan_mask;
2031         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2032                                         (const uint8_t *)&nic_mask,
2033                                         sizeof(struct rte_flow_item_vlan),
2034                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2035         if (ret)
2036                 return ret;
2037         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2038                 struct mlx5_priv *priv = dev->data->dev_private;
2039
2040                 if (priv->vmwa_context) {
2041                         /*
2042                          * Non-NULL context means we have a virtual machine
2043                          * and SR-IOV enabled, we have to create VLAN interface
2044                          * to make hypervisor to setup E-Switch vport
2045                          * context correctly. We avoid creating the multiple
2046                          * VLAN interfaces, so we cannot support VLAN tag mask.
2047                          */
2048                         return rte_flow_error_set(error, EINVAL,
2049                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2050                                                   item,
2051                                                   "VLAN tag mask is not"
2052                                                   " supported in virtual"
2053                                                   " environment");
2054                 }
2055         }
2056         if (spec) {
2057                 vlan_tag = spec->tci;
2058                 vlan_tag &= mask->tci;
2059         }
2060         /*
2061          * From verbs perspective an empty VLAN is equivalent
2062          * to a packet without VLAN layer.
2063          */
2064         if (!vlan_tag)
2065                 return rte_flow_error_set(error, EINVAL,
2066                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2067                                           item->spec,
2068                                           "VLAN cannot be empty");
2069         return 0;
2070 }
2071
2072 /**
2073  * Validate IPV4 item.
2074  *
2075  * @param[in] item
2076  *   Item specification.
2077  * @param[in] item_flags
2078  *   Bit-fields that holds the items detected until now.
2079  * @param[in] last_item
2080  *   Previous validated item in the pattern items.
2081  * @param[in] ether_type
2082  *   Type in the ethernet layer header (including dot1q).
2083  * @param[in] acc_mask
2084  *   Acceptable mask, if NULL default internal default mask
2085  *   will be used to check whether item fields are supported.
2086  * @param[in] range_accepted
2087  *   True if range of values is accepted for specific fields, false otherwise.
2088  * @param[out] error
2089  *   Pointer to error structure.
2090  *
2091  * @return
2092  *   0 on success, a negative errno value otherwise and rte_errno is set.
2093  */
2094 int
2095 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2096                              uint64_t item_flags,
2097                              uint64_t last_item,
2098                              uint16_t ether_type,
2099                              const struct rte_flow_item_ipv4 *acc_mask,
2100                              bool range_accepted,
2101                              struct rte_flow_error *error)
2102 {
2103         const struct rte_flow_item_ipv4 *mask = item->mask;
2104         const struct rte_flow_item_ipv4 *spec = item->spec;
2105         const struct rte_flow_item_ipv4 nic_mask = {
2106                 .hdr = {
2107                         .src_addr = RTE_BE32(0xffffffff),
2108                         .dst_addr = RTE_BE32(0xffffffff),
2109                         .type_of_service = 0xff,
2110                         .next_proto_id = 0xff,
2111                 },
2112         };
2113         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2114         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2115                                       MLX5_FLOW_LAYER_OUTER_L3;
2116         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2117                                       MLX5_FLOW_LAYER_OUTER_L4;
2118         int ret;
2119         uint8_t next_proto = 0xFF;
2120         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2121                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2122                                   MLX5_FLOW_LAYER_INNER_VLAN);
2123
2124         if ((last_item & l2_vlan) && ether_type &&
2125             ether_type != RTE_ETHER_TYPE_IPV4)
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2128                                           "IPv4 cannot follow L2/VLAN layer "
2129                                           "which ether type is not IPv4");
2130         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2131                 if (mask && spec)
2132                         next_proto = mask->hdr.next_proto_id &
2133                                      spec->hdr.next_proto_id;
2134                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2135                         return rte_flow_error_set(error, EINVAL,
2136                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2137                                                   item,
2138                                                   "multiple tunnel "
2139                                                   "not supported");
2140         }
2141         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2142                 return rte_flow_error_set(error, EINVAL,
2143                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2144                                           "wrong tunnel type - IPv6 specified "
2145                                           "but IPv4 item provided");
2146         if (item_flags & l3m)
2147                 return rte_flow_error_set(error, ENOTSUP,
2148                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2149                                           "multiple L3 layers not supported");
2150         else if (item_flags & l4m)
2151                 return rte_flow_error_set(error, EINVAL,
2152                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2153                                           "L3 cannot follow an L4 layer.");
2154         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2155                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2156                 return rte_flow_error_set(error, EINVAL,
2157                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2158                                           "L3 cannot follow an NVGRE layer.");
2159         if (!mask)
2160                 mask = &rte_flow_item_ipv4_mask;
2161         else if (mask->hdr.next_proto_id != 0 &&
2162                  mask->hdr.next_proto_id != 0xff)
2163                 return rte_flow_error_set(error, EINVAL,
2164                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2165                                           "partial mask is not supported"
2166                                           " for protocol");
2167         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2168                                         acc_mask ? (const uint8_t *)acc_mask
2169                                                  : (const uint8_t *)&nic_mask,
2170                                         sizeof(struct rte_flow_item_ipv4),
2171                                         range_accepted, error);
2172         if (ret < 0)
2173                 return ret;
2174         return 0;
2175 }
2176
2177 /**
2178  * Validate IPV6 item.
2179  *
2180  * @param[in] item
2181  *   Item specification.
2182  * @param[in] item_flags
2183  *   Bit-fields that holds the items detected until now.
2184  * @param[in] last_item
2185  *   Previous validated item in the pattern items.
2186  * @param[in] ether_type
2187  *   Type in the ethernet layer header (including dot1q).
2188  * @param[in] acc_mask
2189  *   Acceptable mask, if NULL default internal default mask
2190  *   will be used to check whether item fields are supported.
2191  * @param[out] error
2192  *   Pointer to error structure.
2193  *
2194  * @return
2195  *   0 on success, a negative errno value otherwise and rte_errno is set.
2196  */
2197 int
2198 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2199                              uint64_t item_flags,
2200                              uint64_t last_item,
2201                              uint16_t ether_type,
2202                              const struct rte_flow_item_ipv6 *acc_mask,
2203                              struct rte_flow_error *error)
2204 {
2205         const struct rte_flow_item_ipv6 *mask = item->mask;
2206         const struct rte_flow_item_ipv6 *spec = item->spec;
2207         const struct rte_flow_item_ipv6 nic_mask = {
2208                 .hdr = {
2209                         .src_addr =
2210                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2211                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2212                         .dst_addr =
2213                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2214                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2215                         .vtc_flow = RTE_BE32(0xffffffff),
2216                         .proto = 0xff,
2217                 },
2218         };
2219         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2220         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2221                                       MLX5_FLOW_LAYER_OUTER_L3;
2222         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2223                                       MLX5_FLOW_LAYER_OUTER_L4;
2224         int ret;
2225         uint8_t next_proto = 0xFF;
2226         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2227                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2228                                   MLX5_FLOW_LAYER_INNER_VLAN);
2229
2230         if ((last_item & l2_vlan) && ether_type &&
2231             ether_type != RTE_ETHER_TYPE_IPV6)
2232                 return rte_flow_error_set(error, EINVAL,
2233                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2234                                           "IPv6 cannot follow L2/VLAN layer "
2235                                           "which ether type is not IPv6");
2236         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2237                 next_proto = spec->hdr.proto;
2238         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2239                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2240                         return rte_flow_error_set(error, EINVAL,
2241                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2242                                                   item,
2243                                                   "multiple tunnel "
2244                                                   "not supported");
2245         }
2246         if (next_proto == IPPROTO_HOPOPTS  ||
2247             next_proto == IPPROTO_ROUTING  ||
2248             next_proto == IPPROTO_FRAGMENT ||
2249             next_proto == IPPROTO_ESP      ||
2250             next_proto == IPPROTO_AH       ||
2251             next_proto == IPPROTO_DSTOPTS)
2252                 return rte_flow_error_set(error, EINVAL,
2253                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2254                                           "IPv6 proto (next header) should "
2255                                           "not be set as extension header");
2256         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2257                 return rte_flow_error_set(error, EINVAL,
2258                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2259                                           "wrong tunnel type - IPv4 specified "
2260                                           "but IPv6 item provided");
2261         if (item_flags & l3m)
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2264                                           "multiple L3 layers not supported");
2265         else if (item_flags & l4m)
2266                 return rte_flow_error_set(error, EINVAL,
2267                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2268                                           "L3 cannot follow an L4 layer.");
2269         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2270                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2271                 return rte_flow_error_set(error, EINVAL,
2272                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2273                                           "L3 cannot follow an NVGRE layer.");
2274         if (!mask)
2275                 mask = &rte_flow_item_ipv6_mask;
2276         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2277                                         acc_mask ? (const uint8_t *)acc_mask
2278                                                  : (const uint8_t *)&nic_mask,
2279                                         sizeof(struct rte_flow_item_ipv6),
2280                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2281         if (ret < 0)
2282                 return ret;
2283         return 0;
2284 }
2285
2286 /**
2287  * Validate UDP item.
2288  *
2289  * @param[in] item
2290  *   Item specification.
2291  * @param[in] item_flags
2292  *   Bit-fields that holds the items detected until now.
2293  * @param[in] target_protocol
2294  *   The next protocol in the previous item.
2295  * @param[in] flow_mask
2296  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2297  * @param[out] error
2298  *   Pointer to error structure.
2299  *
2300  * @return
2301  *   0 on success, a negative errno value otherwise and rte_errno is set.
2302  */
2303 int
2304 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2305                             uint64_t item_flags,
2306                             uint8_t target_protocol,
2307                             struct rte_flow_error *error)
2308 {
2309         const struct rte_flow_item_udp *mask = item->mask;
2310         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2311         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2312                                       MLX5_FLOW_LAYER_OUTER_L3;
2313         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2314                                       MLX5_FLOW_LAYER_OUTER_L4;
2315         int ret;
2316
2317         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2318                 return rte_flow_error_set(error, EINVAL,
2319                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2320                                           "protocol filtering not compatible"
2321                                           " with UDP layer");
2322         if (!(item_flags & l3m))
2323                 return rte_flow_error_set(error, EINVAL,
2324                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2325                                           "L3 is mandatory to filter on L4");
2326         if (item_flags & l4m)
2327                 return rte_flow_error_set(error, EINVAL,
2328                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2329                                           "multiple L4 layers not supported");
2330         if (!mask)
2331                 mask = &rte_flow_item_udp_mask;
2332         ret = mlx5_flow_item_acceptable
2333                 (item, (const uint8_t *)mask,
2334                  (const uint8_t *)&rte_flow_item_udp_mask,
2335                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2336                  error);
2337         if (ret < 0)
2338                 return ret;
2339         return 0;
2340 }
2341
2342 /**
2343  * Validate TCP item.
2344  *
2345  * @param[in] item
2346  *   Item specification.
2347  * @param[in] item_flags
2348  *   Bit-fields that holds the items detected until now.
2349  * @param[in] target_protocol
2350  *   The next protocol in the previous item.
2351  * @param[out] error
2352  *   Pointer to error structure.
2353  *
2354  * @return
2355  *   0 on success, a negative errno value otherwise and rte_errno is set.
2356  */
2357 int
2358 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2359                             uint64_t item_flags,
2360                             uint8_t target_protocol,
2361                             const struct rte_flow_item_tcp *flow_mask,
2362                             struct rte_flow_error *error)
2363 {
2364         const struct rte_flow_item_tcp *mask = item->mask;
2365         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2366         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2367                                       MLX5_FLOW_LAYER_OUTER_L3;
2368         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2369                                       MLX5_FLOW_LAYER_OUTER_L4;
2370         int ret;
2371
2372         MLX5_ASSERT(flow_mask);
2373         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2374                 return rte_flow_error_set(error, EINVAL,
2375                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2376                                           "protocol filtering not compatible"
2377                                           " with TCP layer");
2378         if (!(item_flags & l3m))
2379                 return rte_flow_error_set(error, EINVAL,
2380                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2381                                           "L3 is mandatory to filter on L4");
2382         if (item_flags & l4m)
2383                 return rte_flow_error_set(error, EINVAL,
2384                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2385                                           "multiple L4 layers not supported");
2386         if (!mask)
2387                 mask = &rte_flow_item_tcp_mask;
2388         ret = mlx5_flow_item_acceptable
2389                 (item, (const uint8_t *)mask,
2390                  (const uint8_t *)flow_mask,
2391                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2392                  error);
2393         if (ret < 0)
2394                 return ret;
2395         return 0;
2396 }
2397
2398 /**
2399  * Validate VXLAN item.
2400  *
2401  * @param[in] item
2402  *   Item specification.
2403  * @param[in] item_flags
2404  *   Bit-fields that holds the items detected until now.
2405  * @param[in] target_protocol
2406  *   The next protocol in the previous item.
2407  * @param[out] error
2408  *   Pointer to error structure.
2409  *
2410  * @return
2411  *   0 on success, a negative errno value otherwise and rte_errno is set.
2412  */
2413 int
2414 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2415                               uint64_t item_flags,
2416                               struct rte_flow_error *error)
2417 {
2418         const struct rte_flow_item_vxlan *spec = item->spec;
2419         const struct rte_flow_item_vxlan *mask = item->mask;
2420         int ret;
2421         union vni {
2422                 uint32_t vlan_id;
2423                 uint8_t vni[4];
2424         } id = { .vlan_id = 0, };
2425
2426
2427         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2428                 return rte_flow_error_set(error, ENOTSUP,
2429                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2430                                           "multiple tunnel layers not"
2431                                           " supported");
2432         /*
2433          * Verify only UDPv4 is present as defined in
2434          * https://tools.ietf.org/html/rfc7348
2435          */
2436         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2437                 return rte_flow_error_set(error, EINVAL,
2438                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2439                                           "no outer UDP layer found");
2440         if (!mask)
2441                 mask = &rte_flow_item_vxlan_mask;
2442         ret = mlx5_flow_item_acceptable
2443                 (item, (const uint8_t *)mask,
2444                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2445                  sizeof(struct rte_flow_item_vxlan),
2446                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2447         if (ret < 0)
2448                 return ret;
2449         if (spec) {
2450                 memcpy(&id.vni[1], spec->vni, 3);
2451                 memcpy(&id.vni[1], mask->vni, 3);
2452         }
2453         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2454                 return rte_flow_error_set(error, ENOTSUP,
2455                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2456                                           "VXLAN tunnel must be fully defined");
2457         return 0;
2458 }
2459
2460 /**
2461  * Validate VXLAN_GPE item.
2462  *
2463  * @param[in] item
2464  *   Item specification.
2465  * @param[in] item_flags
2466  *   Bit-fields that holds the items detected until now.
2467  * @param[in] priv
2468  *   Pointer to the private data structure.
2469  * @param[in] target_protocol
2470  *   The next protocol in the previous item.
2471  * @param[out] error
2472  *   Pointer to error structure.
2473  *
2474  * @return
2475  *   0 on success, a negative errno value otherwise and rte_errno is set.
2476  */
2477 int
2478 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2479                                   uint64_t item_flags,
2480                                   struct rte_eth_dev *dev,
2481                                   struct rte_flow_error *error)
2482 {
2483         struct mlx5_priv *priv = dev->data->dev_private;
2484         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2485         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2486         int ret;
2487         union vni {
2488                 uint32_t vlan_id;
2489                 uint8_t vni[4];
2490         } id = { .vlan_id = 0, };
2491
2492         if (!priv->config.l3_vxlan_en)
2493                 return rte_flow_error_set(error, ENOTSUP,
2494                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2495                                           "L3 VXLAN is not enabled by device"
2496                                           " parameter and/or not configured in"
2497                                           " firmware");
2498         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2499                 return rte_flow_error_set(error, ENOTSUP,
2500                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2501                                           "multiple tunnel layers not"
2502                                           " supported");
2503         /*
2504          * Verify only UDPv4 is present as defined in
2505          * https://tools.ietf.org/html/rfc7348
2506          */
2507         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2508                 return rte_flow_error_set(error, EINVAL,
2509                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2510                                           "no outer UDP layer found");
2511         if (!mask)
2512                 mask = &rte_flow_item_vxlan_gpe_mask;
2513         ret = mlx5_flow_item_acceptable
2514                 (item, (const uint8_t *)mask,
2515                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2516                  sizeof(struct rte_flow_item_vxlan_gpe),
2517                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2518         if (ret < 0)
2519                 return ret;
2520         if (spec) {
2521                 if (spec->protocol)
2522                         return rte_flow_error_set(error, ENOTSUP,
2523                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2524                                                   item,
2525                                                   "VxLAN-GPE protocol"
2526                                                   " not supported");
2527                 memcpy(&id.vni[1], spec->vni, 3);
2528                 memcpy(&id.vni[1], mask->vni, 3);
2529         }
2530         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2531                 return rte_flow_error_set(error, ENOTSUP,
2532                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2533                                           "VXLAN-GPE tunnel must be fully"
2534                                           " defined");
2535         return 0;
2536 }
2537 /**
2538  * Validate GRE Key item.
2539  *
2540  * @param[in] item
2541  *   Item specification.
2542  * @param[in] item_flags
2543  *   Bit flags to mark detected items.
2544  * @param[in] gre_item
2545  *   Pointer to gre_item
2546  * @param[out] error
2547  *   Pointer to error structure.
2548  *
2549  * @return
2550  *   0 on success, a negative errno value otherwise and rte_errno is set.
2551  */
2552 int
2553 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2554                                 uint64_t item_flags,
2555                                 const struct rte_flow_item *gre_item,
2556                                 struct rte_flow_error *error)
2557 {
2558         const rte_be32_t *mask = item->mask;
2559         int ret = 0;
2560         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2561         const struct rte_flow_item_gre *gre_spec;
2562         const struct rte_flow_item_gre *gre_mask;
2563
2564         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2565                 return rte_flow_error_set(error, ENOTSUP,
2566                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2567                                           "Multiple GRE key not support");
2568         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2569                 return rte_flow_error_set(error, ENOTSUP,
2570                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2571                                           "No preceding GRE header");
2572         if (item_flags & MLX5_FLOW_LAYER_INNER)
2573                 return rte_flow_error_set(error, ENOTSUP,
2574                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2575                                           "GRE key following a wrong item");
2576         gre_mask = gre_item->mask;
2577         if (!gre_mask)
2578                 gre_mask = &rte_flow_item_gre_mask;
2579         gre_spec = gre_item->spec;
2580         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2581                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2582                 return rte_flow_error_set(error, EINVAL,
2583                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2584                                           "Key bit must be on");
2585
2586         if (!mask)
2587                 mask = &gre_key_default_mask;
2588         ret = mlx5_flow_item_acceptable
2589                 (item, (const uint8_t *)mask,
2590                  (const uint8_t *)&gre_key_default_mask,
2591                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2592         return ret;
2593 }
2594
2595 /**
2596  * Validate GRE item.
2597  *
2598  * @param[in] item
2599  *   Item specification.
2600  * @param[in] item_flags
2601  *   Bit flags to mark detected items.
2602  * @param[in] target_protocol
2603  *   The next protocol in the previous item.
2604  * @param[out] error
2605  *   Pointer to error structure.
2606  *
2607  * @return
2608  *   0 on success, a negative errno value otherwise and rte_errno is set.
2609  */
2610 int
2611 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2612                             uint64_t item_flags,
2613                             uint8_t target_protocol,
2614                             struct rte_flow_error *error)
2615 {
2616         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2617         const struct rte_flow_item_gre *mask = item->mask;
2618         int ret;
2619         const struct rte_flow_item_gre nic_mask = {
2620                 .c_rsvd0_ver = RTE_BE16(0xB000),
2621                 .protocol = RTE_BE16(UINT16_MAX),
2622         };
2623
2624         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2627                                           "protocol filtering not compatible"
2628                                           " with this GRE layer");
2629         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2632                                           "multiple tunnel layers not"
2633                                           " supported");
2634         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2635                 return rte_flow_error_set(error, ENOTSUP,
2636                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2637                                           "L3 Layer is missing");
2638         if (!mask)
2639                 mask = &rte_flow_item_gre_mask;
2640         ret = mlx5_flow_item_acceptable
2641                 (item, (const uint8_t *)mask,
2642                  (const uint8_t *)&nic_mask,
2643                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2644                  error);
2645         if (ret < 0)
2646                 return ret;
2647 #ifndef HAVE_MLX5DV_DR
2648 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2649         if (spec && (spec->protocol & mask->protocol))
2650                 return rte_flow_error_set(error, ENOTSUP,
2651                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2652                                           "without MPLS support the"
2653                                           " specification cannot be used for"
2654                                           " filtering");
2655 #endif
2656 #endif
2657         return 0;
2658 }
2659
2660 /**
2661  * Validate Geneve item.
2662  *
2663  * @param[in] item
2664  *   Item specification.
2665  * @param[in] itemFlags
2666  *   Bit-fields that holds the items detected until now.
2667  * @param[in] enPriv
2668  *   Pointer to the private data structure.
2669  * @param[out] error
2670  *   Pointer to error structure.
2671  *
2672  * @return
2673  *   0 on success, a negative errno value otherwise and rte_errno is set.
2674  */
2675
2676 int
2677 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2678                                uint64_t item_flags,
2679                                struct rte_eth_dev *dev,
2680                                struct rte_flow_error *error)
2681 {
2682         struct mlx5_priv *priv = dev->data->dev_private;
2683         const struct rte_flow_item_geneve *spec = item->spec;
2684         const struct rte_flow_item_geneve *mask = item->mask;
2685         int ret;
2686         uint16_t gbhdr;
2687         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2688                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2689         const struct rte_flow_item_geneve nic_mask = {
2690                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2691                 .vni = "\xff\xff\xff",
2692                 .protocol = RTE_BE16(UINT16_MAX),
2693         };
2694
2695         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2696                 return rte_flow_error_set(error, ENOTSUP,
2697                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2698                                           "L3 Geneve is not enabled by device"
2699                                           " parameter and/or not configured in"
2700                                           " firmware");
2701         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2702                 return rte_flow_error_set(error, ENOTSUP,
2703                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2704                                           "multiple tunnel layers not"
2705                                           " supported");
2706         /*
2707          * Verify only UDPv4 is present as defined in
2708          * https://tools.ietf.org/html/rfc7348
2709          */
2710         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2711                 return rte_flow_error_set(error, EINVAL,
2712                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2713                                           "no outer UDP layer found");
2714         if (!mask)
2715                 mask = &rte_flow_item_geneve_mask;
2716         ret = mlx5_flow_item_acceptable
2717                                   (item, (const uint8_t *)mask,
2718                                    (const uint8_t *)&nic_mask,
2719                                    sizeof(struct rte_flow_item_geneve),
2720                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2721         if (ret)
2722                 return ret;
2723         if (spec) {
2724                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2725                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2726                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2727                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2728                         return rte_flow_error_set(error, ENOTSUP,
2729                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2730                                                   item,
2731                                                   "Geneve protocol unsupported"
2732                                                   " fields are being used");
2733                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2734                         return rte_flow_error_set
2735                                         (error, ENOTSUP,
2736                                          RTE_FLOW_ERROR_TYPE_ITEM,
2737                                          item,
2738                                          "Unsupported Geneve options length");
2739         }
2740         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2741                 return rte_flow_error_set
2742                                     (error, ENOTSUP,
2743                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2744                                      "Geneve tunnel must be fully defined");
2745         return 0;
2746 }
2747
2748 /**
2749  * Validate Geneve TLV option item.
2750  *
2751  * @param[in] item
2752  *   Item specification.
2753  * @param[in] last_item
2754  *   Previous validated item in the pattern items.
2755  * @param[in] geneve_item
2756  *   Previous GENEVE item specification.
2757  * @param[in] dev
2758  *   Pointer to the rte_eth_dev structure.
2759  * @param[out] error
2760  *   Pointer to error structure.
2761  *
2762  * @return
2763  *   0 on success, a negative errno value otherwise and rte_errno is set.
2764  */
2765 int
2766 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2767                                    uint64_t last_item,
2768                                    const struct rte_flow_item *geneve_item,
2769                                    struct rte_eth_dev *dev,
2770                                    struct rte_flow_error *error)
2771 {
2772         struct mlx5_priv *priv = dev->data->dev_private;
2773         struct mlx5_dev_ctx_shared *sh = priv->sh;
2774         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2775         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2776         uint8_t data_max_supported =
2777                         hca_attr->max_geneve_tlv_option_data_len * 4;
2778         struct mlx5_dev_config *config = &priv->config;
2779         const struct rte_flow_item_geneve *geneve_spec;
2780         const struct rte_flow_item_geneve *geneve_mask;
2781         const struct rte_flow_item_geneve_opt *spec = item->spec;
2782         const struct rte_flow_item_geneve_opt *mask = item->mask;
2783         unsigned int i;
2784         unsigned int data_len;
2785         uint8_t tlv_option_len;
2786         uint16_t optlen_m, optlen_v;
2787         const struct rte_flow_item_geneve_opt full_mask = {
2788                 .option_class = RTE_BE16(0xffff),
2789                 .option_type = 0xff,
2790                 .option_len = 0x1f,
2791         };
2792
2793         if (!mask)
2794                 mask = &rte_flow_item_geneve_opt_mask;
2795         if (!spec)
2796                 return rte_flow_error_set
2797                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2798                         "Geneve TLV opt class/type/length must be specified");
2799         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2800                 return rte_flow_error_set
2801                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2802                         "Geneve TLV opt length exceeeds the limit (31)");
2803         /* Check if class type and length masks are full. */
2804         if (full_mask.option_class != mask->option_class ||
2805             full_mask.option_type != mask->option_type ||
2806             full_mask.option_len != (mask->option_len & full_mask.option_len))
2807                 return rte_flow_error_set
2808                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2809                         "Geneve TLV opt class/type/length masks must be full");
2810         /* Check if length is supported */
2811         if ((uint32_t)spec->option_len >
2812                         config->hca_attr.max_geneve_tlv_option_data_len)
2813                 return rte_flow_error_set
2814                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2815                         "Geneve TLV opt length not supported");
2816         if (config->hca_attr.max_geneve_tlv_options > 1)
2817                 DRV_LOG(DEBUG,
2818                         "max_geneve_tlv_options supports more than 1 option");
2819         /* Check GENEVE item preceding. */
2820         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2821                 return rte_flow_error_set
2822                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2823                         "Geneve opt item must be preceded with Geneve item");
2824         geneve_spec = geneve_item->spec;
2825         geneve_mask = geneve_item->mask ? geneve_item->mask :
2826                                           &rte_flow_item_geneve_mask;
2827         /* Check if GENEVE TLV option size doesn't exceed option length */
2828         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2829                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2830                 tlv_option_len = spec->option_len & mask->option_len;
2831                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2832                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2833                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2834                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2835                 if ((optlen_v & optlen_m) <= tlv_option_len)
2836                         return rte_flow_error_set
2837                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2838                                  "GENEVE TLV option length exceeds optlen");
2839         }
2840         /* Check if length is 0 or data is 0. */
2841         if (spec->data == NULL || spec->option_len == 0)
2842                 return rte_flow_error_set
2843                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2844                         "Geneve TLV opt with zero data/length not supported");
2845         /* Check not all data & mask are 0. */
2846         data_len = spec->option_len * 4;
2847         if (mask->data == NULL) {
2848                 for (i = 0; i < data_len; i++)
2849                         if (spec->data[i])
2850                                 break;
2851                 if (i == data_len)
2852                         return rte_flow_error_set(error, ENOTSUP,
2853                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2854                                 "Can't match on Geneve option data 0");
2855         } else {
2856                 for (i = 0; i < data_len; i++)
2857                         if (spec->data[i] & mask->data[i])
2858                                 break;
2859                 if (i == data_len)
2860                         return rte_flow_error_set(error, ENOTSUP,
2861                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2862                                 "Can't match on Geneve option data and mask 0");
2863                 /* Check data mask supported. */
2864                 for (i = data_max_supported; i < data_len ; i++)
2865                         if (mask->data[i])
2866                                 return rte_flow_error_set(error, ENOTSUP,
2867                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2868                                         "Data mask is of unsupported size");
2869         }
2870         /* Check GENEVE option is supported in NIC. */
2871         if (!config->hca_attr.geneve_tlv_opt)
2872                 return rte_flow_error_set
2873                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2874                         "Geneve TLV opt not supported");
2875         /* Check if we already have geneve option with different type/class. */
2876         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2877         geneve_opt_resource = sh->geneve_tlv_option_resource;
2878         if (geneve_opt_resource != NULL)
2879                 if (geneve_opt_resource->option_class != spec->option_class ||
2880                     geneve_opt_resource->option_type != spec->option_type ||
2881                     geneve_opt_resource->length != spec->option_len) {
2882                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2883                         return rte_flow_error_set(error, ENOTSUP,
2884                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2885                                 "Only one Geneve TLV option supported");
2886                 }
2887         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2888         return 0;
2889 }
2890
2891 /**
2892  * Validate MPLS item.
2893  *
2894  * @param[in] dev
2895  *   Pointer to the rte_eth_dev structure.
2896  * @param[in] item
2897  *   Item specification.
2898  * @param[in] item_flags
2899  *   Bit-fields that holds the items detected until now.
2900  * @param[in] prev_layer
2901  *   The protocol layer indicated in previous item.
2902  * @param[out] error
2903  *   Pointer to error structure.
2904  *
2905  * @return
2906  *   0 on success, a negative errno value otherwise and rte_errno is set.
2907  */
2908 int
2909 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2910                              const struct rte_flow_item *item __rte_unused,
2911                              uint64_t item_flags __rte_unused,
2912                              uint64_t prev_layer __rte_unused,
2913                              struct rte_flow_error *error)
2914 {
2915 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2916         const struct rte_flow_item_mpls *mask = item->mask;
2917         struct mlx5_priv *priv = dev->data->dev_private;
2918         int ret;
2919
2920         if (!priv->config.mpls_en)
2921                 return rte_flow_error_set(error, ENOTSUP,
2922                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2923                                           "MPLS not supported or"
2924                                           " disabled in firmware"
2925                                           " configuration.");
2926         /* MPLS over UDP, GRE is allowed */
2927         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2928                             MLX5_FLOW_LAYER_GRE |
2929                             MLX5_FLOW_LAYER_GRE_KEY)))
2930                 return rte_flow_error_set(error, EINVAL,
2931                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2932                                           "protocol filtering not compatible"
2933                                           " with MPLS layer");
2934         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2935         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2936             !(item_flags & MLX5_FLOW_LAYER_GRE))
2937                 return rte_flow_error_set(error, ENOTSUP,
2938                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2939                                           "multiple tunnel layers not"
2940                                           " supported");
2941         if (!mask)
2942                 mask = &rte_flow_item_mpls_mask;
2943         ret = mlx5_flow_item_acceptable
2944                 (item, (const uint8_t *)mask,
2945                  (const uint8_t *)&rte_flow_item_mpls_mask,
2946                  sizeof(struct rte_flow_item_mpls),
2947                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2948         if (ret < 0)
2949                 return ret;
2950         return 0;
2951 #else
2952         return rte_flow_error_set(error, ENOTSUP,
2953                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2954                                   "MPLS is not supported by Verbs, please"
2955                                   " update.");
2956 #endif
2957 }
2958
2959 /**
2960  * Validate NVGRE item.
2961  *
2962  * @param[in] item
2963  *   Item specification.
2964  * @param[in] item_flags
2965  *   Bit flags to mark detected items.
2966  * @param[in] target_protocol
2967  *   The next protocol in the previous item.
2968  * @param[out] error
2969  *   Pointer to error structure.
2970  *
2971  * @return
2972  *   0 on success, a negative errno value otherwise and rte_errno is set.
2973  */
2974 int
2975 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2976                               uint64_t item_flags,
2977                               uint8_t target_protocol,
2978                               struct rte_flow_error *error)
2979 {
2980         const struct rte_flow_item_nvgre *mask = item->mask;
2981         int ret;
2982
2983         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2984                 return rte_flow_error_set(error, EINVAL,
2985                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2986                                           "protocol filtering not compatible"
2987                                           " with this GRE layer");
2988         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2989                 return rte_flow_error_set(error, ENOTSUP,
2990                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2991                                           "multiple tunnel layers not"
2992                                           " supported");
2993         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2994                 return rte_flow_error_set(error, ENOTSUP,
2995                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2996                                           "L3 Layer is missing");
2997         if (!mask)
2998                 mask = &rte_flow_item_nvgre_mask;
2999         ret = mlx5_flow_item_acceptable
3000                 (item, (const uint8_t *)mask,
3001                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3002                  sizeof(struct rte_flow_item_nvgre),
3003                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3004         if (ret < 0)
3005                 return ret;
3006         return 0;
3007 }
3008
3009 /**
3010  * Validate eCPRI item.
3011  *
3012  * @param[in] item
3013  *   Item specification.
3014  * @param[in] item_flags
3015  *   Bit-fields that holds the items detected until now.
3016  * @param[in] last_item
3017  *   Previous validated item in the pattern items.
3018  * @param[in] ether_type
3019  *   Type in the ethernet layer header (including dot1q).
3020  * @param[in] acc_mask
3021  *   Acceptable mask, if NULL default internal default mask
3022  *   will be used to check whether item fields are supported.
3023  * @param[out] error
3024  *   Pointer to error structure.
3025  *
3026  * @return
3027  *   0 on success, a negative errno value otherwise and rte_errno is set.
3028  */
3029 int
3030 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3031                               uint64_t item_flags,
3032                               uint64_t last_item,
3033                               uint16_t ether_type,
3034                               const struct rte_flow_item_ecpri *acc_mask,
3035                               struct rte_flow_error *error)
3036 {
3037         const struct rte_flow_item_ecpri *mask = item->mask;
3038         const struct rte_flow_item_ecpri nic_mask = {
3039                 .hdr = {
3040                         .common = {
3041                                 .u32 =
3042                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3043                                         .type = 0xFF,
3044                                         }).u32),
3045                         },
3046                         .dummy[0] = 0xFFFFFFFF,
3047                 },
3048         };
3049         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3050                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3051         struct rte_flow_item_ecpri mask_lo;
3052
3053         if (!(last_item & outer_l2_vlan) &&
3054             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3055                 return rte_flow_error_set(error, EINVAL,
3056                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3057                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3058         if ((last_item & outer_l2_vlan) && ether_type &&
3059             ether_type != RTE_ETHER_TYPE_ECPRI)
3060                 return rte_flow_error_set(error, EINVAL,
3061                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3062                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3063         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3064                 return rte_flow_error_set(error, EINVAL,
3065                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3066                                           "eCPRI with tunnel is not supported right now");
3067         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3068                 return rte_flow_error_set(error, ENOTSUP,
3069                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3070                                           "multiple L3 layers not supported");
3071         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3072                 return rte_flow_error_set(error, EINVAL,
3073                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3074                                           "eCPRI cannot coexist with a TCP layer");
3075         /* In specification, eCPRI could be over UDP layer. */
3076         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3077                 return rte_flow_error_set(error, EINVAL,
3078                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3079                                           "eCPRI over UDP layer is not yet supported right now");
3080         /* Mask for type field in common header could be zero. */
3081         if (!mask)
3082                 mask = &rte_flow_item_ecpri_mask;
3083         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3084         /* Input mask is in big-endian format. */
3085         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3086                 return rte_flow_error_set(error, EINVAL,
3087                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3088                                           "partial mask is not supported for protocol");
3089         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3090                 return rte_flow_error_set(error, EINVAL,
3091                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3092                                           "message header mask must be after a type mask");
3093         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3094                                          acc_mask ? (const uint8_t *)acc_mask
3095                                                   : (const uint8_t *)&nic_mask,
3096                                          sizeof(struct rte_flow_item_ecpri),
3097                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3098 }
3099
3100 /**
3101  * Release resource related QUEUE/RSS action split.
3102  *
3103  * @param dev
3104  *   Pointer to Ethernet device.
3105  * @param flow
3106  *   Flow to release id's from.
3107  */
3108 static void
3109 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
3110                              struct rte_flow *flow)
3111 {
3112         struct mlx5_priv *priv = dev->data->dev_private;
3113         uint32_t handle_idx;
3114         struct mlx5_flow_handle *dev_handle;
3115
3116         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
3117                        handle_idx, dev_handle, next)
3118                 if (dev_handle->split_flow_id &&
3119                     !dev_handle->is_meter_flow_id)
3120                         mlx5_ipool_free(priv->sh->ipool
3121                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
3122                                         dev_handle->split_flow_id);
3123 }
3124
3125 static int
3126 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3127                    const struct rte_flow_attr *attr __rte_unused,
3128                    const struct rte_flow_item items[] __rte_unused,
3129                    const struct rte_flow_action actions[] __rte_unused,
3130                    bool external __rte_unused,
3131                    int hairpin __rte_unused,
3132                    struct rte_flow_error *error)
3133 {
3134         return rte_flow_error_set(error, ENOTSUP,
3135                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3136 }
3137
3138 static struct mlx5_flow *
3139 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3140                   const struct rte_flow_attr *attr __rte_unused,
3141                   const struct rte_flow_item items[] __rte_unused,
3142                   const struct rte_flow_action actions[] __rte_unused,
3143                   struct rte_flow_error *error)
3144 {
3145         rte_flow_error_set(error, ENOTSUP,
3146                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3147         return NULL;
3148 }
3149
3150 static int
3151 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3152                     struct mlx5_flow *dev_flow __rte_unused,
3153                     const struct rte_flow_attr *attr __rte_unused,
3154                     const struct rte_flow_item items[] __rte_unused,
3155                     const struct rte_flow_action actions[] __rte_unused,
3156                     struct rte_flow_error *error)
3157 {
3158         return rte_flow_error_set(error, ENOTSUP,
3159                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3160 }
3161
3162 static int
3163 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3164                 struct rte_flow *flow __rte_unused,
3165                 struct rte_flow_error *error)
3166 {
3167         return rte_flow_error_set(error, ENOTSUP,
3168                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3169 }
3170
3171 static void
3172 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3173                  struct rte_flow *flow __rte_unused)
3174 {
3175 }
3176
3177 static void
3178 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3179                   struct rte_flow *flow __rte_unused)
3180 {
3181 }
3182
3183 static int
3184 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3185                 struct rte_flow *flow __rte_unused,
3186                 const struct rte_flow_action *actions __rte_unused,
3187                 void *data __rte_unused,
3188                 struct rte_flow_error *error)
3189 {
3190         return rte_flow_error_set(error, ENOTSUP,
3191                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3192 }
3193
3194 static int
3195 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3196                       uint32_t domains __rte_unused,
3197                       uint32_t flags __rte_unused)
3198 {
3199         return 0;
3200 }
3201
3202 /* Void driver to protect from null pointer reference. */
3203 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3204         .validate = flow_null_validate,
3205         .prepare = flow_null_prepare,
3206         .translate = flow_null_translate,
3207         .apply = flow_null_apply,
3208         .remove = flow_null_remove,
3209         .destroy = flow_null_destroy,
3210         .query = flow_null_query,
3211         .sync_domain = flow_null_sync_domain,
3212 };
3213
3214 /**
3215  * Select flow driver type according to flow attributes and device
3216  * configuration.
3217  *
3218  * @param[in] dev
3219  *   Pointer to the dev structure.
3220  * @param[in] attr
3221  *   Pointer to the flow attributes.
3222  *
3223  * @return
3224  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3225  */
3226 static enum mlx5_flow_drv_type
3227 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3228 {
3229         struct mlx5_priv *priv = dev->data->dev_private;
3230         /* The OS can determine first a specific flow type (DV, VERBS) */
3231         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3232
3233         if (type != MLX5_FLOW_TYPE_MAX)
3234                 return type;
3235         /* If no OS specific type - continue with DV/VERBS selection */
3236         if (attr->transfer && priv->config.dv_esw_en)
3237                 type = MLX5_FLOW_TYPE_DV;
3238         if (!attr->transfer)
3239                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3240                                                  MLX5_FLOW_TYPE_VERBS;
3241         return type;
3242 }
3243
3244 #define flow_get_drv_ops(type) flow_drv_ops[type]
3245
3246 /**
3247  * Flow driver validation API. This abstracts calling driver specific functions.
3248  * The type of flow driver is determined according to flow attributes.
3249  *
3250  * @param[in] dev
3251  *   Pointer to the dev structure.
3252  * @param[in] attr
3253  *   Pointer to the flow attributes.
3254  * @param[in] items
3255  *   Pointer to the list of items.
3256  * @param[in] actions
3257  *   Pointer to the list of actions.
3258  * @param[in] external
3259  *   This flow rule is created by request external to PMD.
3260  * @param[in] hairpin
3261  *   Number of hairpin TX actions, 0 means classic flow.
3262  * @param[out] error
3263  *   Pointer to the error structure.
3264  *
3265  * @return
3266  *   0 on success, a negative errno value otherwise and rte_errno is set.
3267  */
3268 static inline int
3269 flow_drv_validate(struct rte_eth_dev *dev,
3270                   const struct rte_flow_attr *attr,
3271                   const struct rte_flow_item items[],
3272                   const struct rte_flow_action actions[],
3273                   bool external, int hairpin, struct rte_flow_error *error)
3274 {
3275         const struct mlx5_flow_driver_ops *fops;
3276         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3277
3278         fops = flow_get_drv_ops(type);
3279         return fops->validate(dev, attr, items, actions, external,
3280                               hairpin, error);
3281 }
3282
3283 /**
3284  * Flow driver preparation API. This abstracts calling driver specific
3285  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3286  * calculates the size of memory required for device flow, allocates the memory,
3287  * initializes the device flow and returns the pointer.
3288  *
3289  * @note
3290  *   This function initializes device flow structure such as dv or verbs in
3291  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3292  *   rest. For example, adding returning device flow to flow->dev_flow list and
3293  *   setting backward reference to the flow should be done out of this function.
3294  *   layers field is not filled either.
3295  *
3296  * @param[in] dev
3297  *   Pointer to the dev structure.
3298  * @param[in] attr
3299  *   Pointer to the flow attributes.
3300  * @param[in] items
3301  *   Pointer to the list of items.
3302  * @param[in] actions
3303  *   Pointer to the list of actions.
3304  * @param[in] flow_idx
3305  *   This memory pool index to the flow.
3306  * @param[out] error
3307  *   Pointer to the error structure.
3308  *
3309  * @return
3310  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3311  */
3312 static inline struct mlx5_flow *
3313 flow_drv_prepare(struct rte_eth_dev *dev,
3314                  const struct rte_flow *flow,
3315                  const struct rte_flow_attr *attr,
3316                  const struct rte_flow_item items[],
3317                  const struct rte_flow_action actions[],
3318                  uint32_t flow_idx,
3319                  struct rte_flow_error *error)
3320 {
3321         const struct mlx5_flow_driver_ops *fops;
3322         enum mlx5_flow_drv_type type = flow->drv_type;
3323         struct mlx5_flow *mlx5_flow = NULL;
3324
3325         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3326         fops = flow_get_drv_ops(type);
3327         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3328         if (mlx5_flow)
3329                 mlx5_flow->flow_idx = flow_idx;
3330         return mlx5_flow;
3331 }
3332
3333 /**
3334  * Flow driver translation API. This abstracts calling driver specific
3335  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3336  * translates a generic flow into a driver flow. flow_drv_prepare() must
3337  * precede.
3338  *
3339  * @note
3340  *   dev_flow->layers could be filled as a result of parsing during translation
3341  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3342  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3343  *   flow->actions could be overwritten even though all the expanded dev_flows
3344  *   have the same actions.
3345  *
3346  * @param[in] dev
3347  *   Pointer to the rte dev structure.
3348  * @param[in, out] dev_flow
3349  *   Pointer to the mlx5 flow.
3350  * @param[in] attr
3351  *   Pointer to the flow attributes.
3352  * @param[in] items
3353  *   Pointer to the list of items.
3354  * @param[in] actions
3355  *   Pointer to the list of actions.
3356  * @param[out] error
3357  *   Pointer to the error structure.
3358  *
3359  * @return
3360  *   0 on success, a negative errno value otherwise and rte_errno is set.
3361  */
3362 static inline int
3363 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3364                    const struct rte_flow_attr *attr,
3365                    const struct rte_flow_item items[],
3366                    const struct rte_flow_action actions[],
3367                    struct rte_flow_error *error)
3368 {
3369         const struct mlx5_flow_driver_ops *fops;
3370         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3371
3372         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3373         fops = flow_get_drv_ops(type);
3374         return fops->translate(dev, dev_flow, attr, items, actions, error);
3375 }
3376
3377 /**
3378  * Flow driver apply API. This abstracts calling driver specific functions.
3379  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3380  * translated driver flows on to device. flow_drv_translate() must precede.
3381  *
3382  * @param[in] dev
3383  *   Pointer to Ethernet device structure.
3384  * @param[in, out] flow
3385  *   Pointer to flow structure.
3386  * @param[out] error
3387  *   Pointer to error structure.
3388  *
3389  * @return
3390  *   0 on success, a negative errno value otherwise and rte_errno is set.
3391  */
3392 static inline int
3393 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3394                struct rte_flow_error *error)
3395 {
3396         const struct mlx5_flow_driver_ops *fops;
3397         enum mlx5_flow_drv_type type = flow->drv_type;
3398
3399         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3400         fops = flow_get_drv_ops(type);
3401         return fops->apply(dev, flow, error);
3402 }
3403
3404 /**
3405  * Flow driver destroy API. This abstracts calling driver specific functions.
3406  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3407  * on device and releases resources of the flow.
3408  *
3409  * @param[in] dev
3410  *   Pointer to Ethernet device.
3411  * @param[in, out] flow
3412  *   Pointer to flow structure.
3413  */
3414 static inline void
3415 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3416 {
3417         const struct mlx5_flow_driver_ops *fops;
3418         enum mlx5_flow_drv_type type = flow->drv_type;
3419
3420         flow_mreg_split_qrss_release(dev, flow);
3421         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3422         fops = flow_get_drv_ops(type);
3423         fops->destroy(dev, flow);
3424 }
3425
3426 /**
3427  * Flow driver find RSS policy tbl API. This abstracts calling driver
3428  * specific functions. Parent flow (rte_flow) should have driver
3429  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3430  *
3431  * @param[in] dev
3432  *   Pointer to Ethernet device.
3433  * @param[in, out] flow
3434  *   Pointer to flow structure.
3435  * @param[in] policy
3436  *   Pointer to meter policy table.
3437  * @param[in] rss_desc
3438  *   Pointer to rss_desc
3439  */
3440 static struct mlx5_flow_meter_sub_policy *
3441 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3442                 struct rte_flow *flow,
3443                 struct mlx5_flow_meter_policy *policy,
3444                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3445 {
3446         const struct mlx5_flow_driver_ops *fops;
3447         enum mlx5_flow_drv_type type = flow->drv_type;
3448
3449         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3450         fops = flow_get_drv_ops(type);
3451         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3452 }
3453
3454 /**
3455  * Get RSS action from the action list.
3456  *
3457  * @param[in] dev
3458  *   Pointer to Ethernet device.
3459  * @param[in] actions
3460  *   Pointer to the list of actions.
3461  * @param[in] flow
3462  *   Parent flow structure pointer.
3463  *
3464  * @return
3465  *   Pointer to the RSS action if exist, else return NULL.
3466  */
3467 static const struct rte_flow_action_rss*
3468 flow_get_rss_action(struct rte_eth_dev *dev,
3469                     const struct rte_flow_action actions[])
3470 {
3471         struct mlx5_priv *priv = dev->data->dev_private;
3472         const struct rte_flow_action_rss *rss = NULL;
3473
3474         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3475                 switch (actions->type) {
3476                 case RTE_FLOW_ACTION_TYPE_RSS:
3477                         rss = actions->conf;
3478                         break;
3479                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3480                 {
3481                         const struct rte_flow_action_sample *sample =
3482                                                                 actions->conf;
3483                         const struct rte_flow_action *act = sample->actions;
3484                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3485                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3486                                         rss = act->conf;
3487                         break;
3488                 }
3489                 case RTE_FLOW_ACTION_TYPE_METER:
3490                 {
3491                         uint32_t mtr_idx;
3492                         struct mlx5_flow_meter_info *fm;
3493                         struct mlx5_flow_meter_policy *policy;
3494                         const struct rte_flow_action_meter *mtr = actions->conf;
3495
3496                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3497                         if (fm) {
3498                                 policy = mlx5_flow_meter_policy_find(dev,
3499                                                 fm->policy_id, NULL);
3500                                 if (policy && policy->is_rss)
3501                                         rss =
3502                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3503                         }
3504                         break;
3505                 }
3506                 default:
3507                         break;
3508                 }
3509         }
3510         return rss;
3511 }
3512
3513 /**
3514  * Get ASO age action by index.
3515  *
3516  * @param[in] dev
3517  *   Pointer to the Ethernet device structure.
3518  * @param[in] age_idx
3519  *   Index to the ASO age action.
3520  *
3521  * @return
3522  *   The specified ASO age action.
3523  */
3524 struct mlx5_aso_age_action*
3525 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3526 {
3527         uint16_t pool_idx = age_idx & UINT16_MAX;
3528         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3529         struct mlx5_priv *priv = dev->data->dev_private;
3530         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3531         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3532
3533         return &pool->actions[offset - 1];
3534 }
3535
3536 /* maps indirect action to translated direct in some actions array */
3537 struct mlx5_translated_action_handle {
3538         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3539         int index; /**< Index in related array of rte_flow_action. */
3540 };
3541
3542 /**
3543  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3544  * direct action if translation possible.
3545  * This functionality used to run same execution path for both direct and
3546  * indirect actions on flow create. All necessary preparations for indirect
3547  * action handling should be performed on *handle* actions list returned
3548  * from this call.
3549  *
3550  * @param[in] dev
3551  *   Pointer to Ethernet device.
3552  * @param[in] actions
3553  *   List of actions to translate.
3554  * @param[out] handle
3555  *   List to store translated indirect action object handles.
3556  * @param[in, out] indir_n
3557  *   Size of *handle* array. On return should be updated with number of
3558  *   indirect actions retrieved from the *actions* list.
3559  * @param[out] translated_actions
3560  *   List of actions where all indirect actions were translated to direct
3561  *   if possible. NULL if no translation took place.
3562  * @param[out] error
3563  *   Pointer to the error structure.
3564  *
3565  * @return
3566  *   0 on success, a negative errno value otherwise and rte_errno is set.
3567  */
3568 static int
3569 flow_action_handles_translate(struct rte_eth_dev *dev,
3570                               const struct rte_flow_action actions[],
3571                               struct mlx5_translated_action_handle *handle,
3572                               int *indir_n,
3573                               struct rte_flow_action **translated_actions,
3574                               struct rte_flow_error *error)
3575 {
3576         struct mlx5_priv *priv = dev->data->dev_private;
3577         struct rte_flow_action *translated = NULL;
3578         size_t actions_size;
3579         int n;
3580         int copied_n = 0;
3581         struct mlx5_translated_action_handle *handle_end = NULL;
3582
3583         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3584                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3585                         continue;
3586                 if (copied_n == *indir_n) {
3587                         return rte_flow_error_set
3588                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3589                                  NULL, "too many shared actions");
3590                 }
3591                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3592                            sizeof(actions[n].conf));
3593                 handle[copied_n].index = n;
3594                 copied_n++;
3595         }
3596         n++;
3597         *indir_n = copied_n;
3598         if (!copied_n)
3599                 return 0;
3600         actions_size = sizeof(struct rte_flow_action) * n;
3601         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3602         if (!translated) {
3603                 rte_errno = ENOMEM;
3604                 return -ENOMEM;
3605         }
3606         memcpy(translated, actions, actions_size);
3607         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3608                 struct mlx5_shared_action_rss *shared_rss;
3609                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3610                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3611                 uint32_t idx = act_idx &
3612                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3613
3614                 switch (type) {
3615                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3616                         shared_rss = mlx5_ipool_get
3617                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3618                         translated[handle->index].type =
3619                                 RTE_FLOW_ACTION_TYPE_RSS;
3620                         translated[handle->index].conf =
3621                                 &shared_rss->origin;
3622                         break;
3623                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3624                         translated[handle->index].type =
3625                                                 (enum rte_flow_action_type)
3626                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3627                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3628                         break;
3629                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3630                         if (priv->sh->flow_hit_aso_en) {
3631                                 translated[handle->index].type =
3632                                         (enum rte_flow_action_type)
3633                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3634                                 translated[handle->index].conf =
3635                                                          (void *)(uintptr_t)idx;
3636                                 break;
3637                         }
3638                         /* Fall-through */
3639                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3640                         if (priv->sh->ct_aso_en) {
3641                                 translated[handle->index].type =
3642                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3643                                 translated[handle->index].conf =
3644                                                          (void *)(uintptr_t)idx;
3645                                 break;
3646                         }
3647                         /* Fall-through */
3648                 default:
3649                         mlx5_free(translated);
3650                         return rte_flow_error_set
3651                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3652                                  NULL, "invalid indirect action type");
3653                 }
3654         }
3655         *translated_actions = translated;
3656         return 0;
3657 }
3658
3659 /**
3660  * Get Shared RSS action from the action list.
3661  *
3662  * @param[in] dev
3663  *   Pointer to Ethernet device.
3664  * @param[in] shared
3665  *   Pointer to the list of actions.
3666  * @param[in] shared_n
3667  *   Actions list length.
3668  *
3669  * @return
3670  *   The MLX5 RSS action ID if exists, otherwise return 0.
3671  */
3672 static uint32_t
3673 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3674                            struct mlx5_translated_action_handle *handle,
3675                            int shared_n)
3676 {
3677         struct mlx5_translated_action_handle *handle_end;
3678         struct mlx5_priv *priv = dev->data->dev_private;
3679         struct mlx5_shared_action_rss *shared_rss;
3680
3681
3682         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3683                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3684                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3685                 uint32_t idx = act_idx &
3686                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3687                 switch (type) {
3688                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3689                         shared_rss = mlx5_ipool_get
3690                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3691                                                                            idx);
3692                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3693                                            __ATOMIC_RELAXED);
3694                         return idx;
3695                 default:
3696                         break;
3697                 }
3698         }
3699         return 0;
3700 }
3701
3702 static unsigned int
3703 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3704 {
3705         const struct rte_flow_item *item;
3706         unsigned int has_vlan = 0;
3707
3708         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3709                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3710                         has_vlan = 1;
3711                         break;
3712                 }
3713         }
3714         if (has_vlan)
3715                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3716                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3717         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3718                                MLX5_EXPANSION_ROOT_OUTER;
3719 }
3720
3721 /**
3722  *  Get layer flags from the prefix flow.
3723  *
3724  *  Some flows may be split to several subflows, the prefix subflow gets the
3725  *  match items and the suffix sub flow gets the actions.
3726  *  Some actions need the user defined match item flags to get the detail for
3727  *  the action.
3728  *  This function helps the suffix flow to get the item layer flags from prefix
3729  *  subflow.
3730  *
3731  * @param[in] dev_flow
3732  *   Pointer the created preifx subflow.
3733  *
3734  * @return
3735  *   The layers get from prefix subflow.
3736  */
3737 static inline uint64_t
3738 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3739 {
3740         uint64_t layers = 0;
3741
3742         /*
3743          * Layers bits could be localization, but usually the compiler will
3744          * help to do the optimization work for source code.
3745          * If no decap actions, use the layers directly.
3746          */
3747         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3748                 return dev_flow->handle->layers;
3749         /* Convert L3 layers with decap action. */
3750         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3751                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3752         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3753                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3754         /* Convert L4 layers with decap action.  */
3755         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3756                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3757         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3758                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3759         return layers;
3760 }
3761
3762 /**
3763  * Get metadata split action information.
3764  *
3765  * @param[in] actions
3766  *   Pointer to the list of actions.
3767  * @param[out] qrss
3768  *   Pointer to the return pointer.
3769  * @param[out] qrss_type
3770  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3771  *   if no QUEUE/RSS is found.
3772  * @param[out] encap_idx
3773  *   Pointer to the index of the encap action if exists, otherwise the last
3774  *   action index.
3775  *
3776  * @return
3777  *   Total number of actions.
3778  */
3779 static int
3780 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3781                                        const struct rte_flow_action **qrss,
3782                                        int *encap_idx)
3783 {
3784         const struct rte_flow_action_raw_encap *raw_encap;
3785         int actions_n = 0;
3786         int raw_decap_idx = -1;
3787
3788         *encap_idx = -1;
3789         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3790                 switch (actions->type) {
3791                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3792                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3793                         *encap_idx = actions_n;
3794                         break;
3795                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3796                         raw_decap_idx = actions_n;
3797                         break;
3798                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3799                         raw_encap = actions->conf;
3800                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3801                                 *encap_idx = raw_decap_idx != -1 ?
3802                                                       raw_decap_idx : actions_n;
3803                         break;
3804                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3805                 case RTE_FLOW_ACTION_TYPE_RSS:
3806                         *qrss = actions;
3807                         break;
3808                 default:
3809                         break;
3810                 }
3811                 actions_n++;
3812         }
3813         if (*encap_idx == -1)
3814                 *encap_idx = actions_n;
3815         /* Count RTE_FLOW_ACTION_TYPE_END. */
3816         return actions_n + 1;
3817 }
3818
3819 /**
3820  * Check if the action will change packet.
3821  *
3822  * @param dev
3823  *   Pointer to Ethernet device.
3824  * @param[in] type
3825  *   action type.
3826  *
3827  * @return
3828  *   true if action will change packet, false otherwise.
3829  */
3830 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3831                                           enum rte_flow_action_type type)
3832 {
3833         struct mlx5_priv *priv = dev->data->dev_private;
3834
3835         switch (type) {
3836         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3837         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3838         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3839         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3840         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3841         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3842         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3843         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3844         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3845         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3846         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3847         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3848         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3849         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3850         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3851         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3852         case RTE_FLOW_ACTION_TYPE_SET_META:
3853         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3854         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3855         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3856         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3857         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3858         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3859         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3860         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3861         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3862         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3863         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3864         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3865                 return true;
3866         case RTE_FLOW_ACTION_TYPE_FLAG:
3867         case RTE_FLOW_ACTION_TYPE_MARK:
3868                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3869                         return true;
3870                 else
3871                         return false;
3872         default:
3873                 return false;
3874         }
3875 }
3876
3877 /**
3878  * Check meter action from the action list.
3879  *
3880  * @param dev
3881  *   Pointer to Ethernet device.
3882  * @param[in] actions
3883  *   Pointer to the list of actions.
3884  * @param[out] has_mtr
3885  *   Pointer to the meter exist flag.
3886  * @param[out] has_modify
3887  *   Pointer to the flag showing there's packet change action.
3888  * @param[out] meter_id
3889  *   Pointer to the meter id.
3890  *
3891  * @return
3892  *   Total number of actions.
3893  */
3894 static int
3895 flow_check_meter_action(struct rte_eth_dev *dev,
3896                         const struct rte_flow_action actions[],
3897                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3898 {
3899         const struct rte_flow_action_meter *mtr = NULL;
3900         int actions_n = 0;
3901
3902         MLX5_ASSERT(has_mtr);
3903         *has_mtr = false;
3904         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3905                 switch (actions->type) {
3906                 case RTE_FLOW_ACTION_TYPE_METER:
3907                         mtr = actions->conf;
3908                         *meter_id = mtr->mtr_id;
3909                         *has_mtr = true;
3910                         break;
3911                 default:
3912                         break;
3913                 }
3914                 if (!*has_mtr)
3915                         *has_modify |= flow_check_modify_action_type(dev,
3916                                                                 actions->type);
3917                 actions_n++;
3918         }
3919         /* Count RTE_FLOW_ACTION_TYPE_END. */
3920         return actions_n + 1;
3921 }
3922
3923 /**
3924  * Check if the flow should be split due to hairpin.
3925  * The reason for the split is that in current HW we can't
3926  * support encap and push-vlan on Rx, so if a flow contains
3927  * these actions we move it to Tx.
3928  *
3929  * @param dev
3930  *   Pointer to Ethernet device.
3931  * @param[in] attr
3932  *   Flow rule attributes.
3933  * @param[in] actions
3934  *   Associated actions (list terminated by the END action).
3935  *
3936  * @return
3937  *   > 0 the number of actions and the flow should be split,
3938  *   0 when no split required.
3939  */
3940 static int
3941 flow_check_hairpin_split(struct rte_eth_dev *dev,
3942                          const struct rte_flow_attr *attr,
3943                          const struct rte_flow_action actions[])
3944 {
3945         int queue_action = 0;
3946         int action_n = 0;
3947         int split = 0;
3948         const struct rte_flow_action_queue *queue;
3949         const struct rte_flow_action_rss *rss;
3950         const struct rte_flow_action_raw_encap *raw_encap;
3951         const struct rte_eth_hairpin_conf *conf;
3952
3953         if (!attr->ingress)
3954                 return 0;
3955         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3956                 switch (actions->type) {
3957                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3958                         queue = actions->conf;
3959                         if (queue == NULL)
3960                                 return 0;
3961                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3962                         if (conf == NULL || conf->tx_explicit != 0)
3963                                 return 0;
3964                         queue_action = 1;
3965                         action_n++;
3966                         break;
3967                 case RTE_FLOW_ACTION_TYPE_RSS:
3968                         rss = actions->conf;
3969                         if (rss == NULL || rss->queue_num == 0)
3970                                 return 0;
3971                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3972                         if (conf == NULL || conf->tx_explicit != 0)
3973                                 return 0;
3974                         queue_action = 1;
3975                         action_n++;
3976                         break;
3977                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3978                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3979                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3980                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3981                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3982                         split++;
3983                         action_n++;
3984                         break;
3985                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3986                         raw_encap = actions->conf;
3987                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3988                                 split++;
3989                         action_n++;
3990                         break;
3991                 default:
3992                         action_n++;
3993                         break;
3994                 }
3995         }
3996         if (split && queue_action)
3997                 return action_n;
3998         return 0;
3999 }
4000
4001 /* Declare flow create/destroy prototype in advance. */
4002 static uint32_t
4003 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
4004                  const struct rte_flow_attr *attr,
4005                  const struct rte_flow_item items[],
4006                  const struct rte_flow_action actions[],
4007                  bool external, struct rte_flow_error *error);
4008
4009 static void
4010 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
4011                   uint32_t flow_idx);
4012
4013 int
4014 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
4015                       struct mlx5_hlist_entry *entry,
4016                       uint64_t key, void *cb_ctx __rte_unused)
4017 {
4018         struct mlx5_flow_mreg_copy_resource *mcp_res =
4019                 container_of(entry, typeof(*mcp_res), hlist_ent);
4020
4021         return mcp_res->mark_id != key;
4022 }
4023
4024 struct mlx5_hlist_entry *
4025 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
4026                        void *cb_ctx)
4027 {
4028         struct rte_eth_dev *dev = list->ctx;
4029         struct mlx5_priv *priv = dev->data->dev_private;
4030         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4031         struct mlx5_flow_mreg_copy_resource *mcp_res;
4032         struct rte_flow_error *error = ctx->error;
4033         uint32_t idx = 0;
4034         int ret;
4035         uint32_t mark_id = key;
4036         struct rte_flow_attr attr = {
4037                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4038                 .ingress = 1,
4039         };
4040         struct mlx5_rte_flow_item_tag tag_spec = {
4041                 .data = mark_id,
4042         };
4043         struct rte_flow_item items[] = {
4044                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4045         };
4046         struct rte_flow_action_mark ftag = {
4047                 .id = mark_id,
4048         };
4049         struct mlx5_flow_action_copy_mreg cp_mreg = {
4050                 .dst = REG_B,
4051                 .src = REG_NON,
4052         };
4053         struct rte_flow_action_jump jump = {
4054                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4055         };
4056         struct rte_flow_action actions[] = {
4057                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4058         };
4059
4060         /* Fill the register fileds in the flow. */
4061         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4062         if (ret < 0)
4063                 return NULL;
4064         tag_spec.id = ret;
4065         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4066         if (ret < 0)
4067                 return NULL;
4068         cp_mreg.src = ret;
4069         /* Provide the full width of FLAG specific value. */
4070         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4071                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4072         /* Build a new flow. */
4073         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4074                 items[0] = (struct rte_flow_item){
4075                         .type = (enum rte_flow_item_type)
4076                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4077                         .spec = &tag_spec,
4078                 };
4079                 items[1] = (struct rte_flow_item){
4080                         .type = RTE_FLOW_ITEM_TYPE_END,
4081                 };
4082                 actions[0] = (struct rte_flow_action){
4083                         .type = (enum rte_flow_action_type)
4084                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4085                         .conf = &ftag,
4086                 };
4087                 actions[1] = (struct rte_flow_action){
4088                         .type = (enum rte_flow_action_type)
4089                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4090                         .conf = &cp_mreg,
4091                 };
4092                 actions[2] = (struct rte_flow_action){
4093                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4094                         .conf = &jump,
4095                 };
4096                 actions[3] = (struct rte_flow_action){
4097                         .type = RTE_FLOW_ACTION_TYPE_END,
4098                 };
4099         } else {
4100                 /* Default rule, wildcard match. */
4101                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4102                 items[0] = (struct rte_flow_item){
4103                         .type = RTE_FLOW_ITEM_TYPE_END,
4104                 };
4105                 actions[0] = (struct rte_flow_action){
4106                         .type = (enum rte_flow_action_type)
4107                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4108                         .conf = &cp_mreg,
4109                 };
4110                 actions[1] = (struct rte_flow_action){
4111                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4112                         .conf = &jump,
4113                 };
4114                 actions[2] = (struct rte_flow_action){
4115                         .type = RTE_FLOW_ACTION_TYPE_END,
4116                 };
4117         }
4118         /* Build a new entry. */
4119         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4120         if (!mcp_res) {
4121                 rte_errno = ENOMEM;
4122                 return NULL;
4123         }
4124         mcp_res->idx = idx;
4125         mcp_res->mark_id = mark_id;
4126         /*
4127          * The copy Flows are not included in any list. There
4128          * ones are referenced from other Flows and can not
4129          * be applied, removed, deleted in ardbitrary order
4130          * by list traversing.
4131          */
4132         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
4133                                          actions, false, error);
4134         if (!mcp_res->rix_flow) {
4135                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4136                 return NULL;
4137         }
4138         return &mcp_res->hlist_ent;
4139 }
4140
4141 /**
4142  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4143  *
4144  * As mark_id is unique, if there's already a registered flow for the mark_id,
4145  * return by increasing the reference counter of the resource. Otherwise, create
4146  * the resource (mcp_res) and flow.
4147  *
4148  * Flow looks like,
4149  *   - If ingress port is ANY and reg_c[1] is mark_id,
4150  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4151  *
4152  * For default flow (zero mark_id), flow is like,
4153  *   - If ingress port is ANY,
4154  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4155  *
4156  * @param dev
4157  *   Pointer to Ethernet device.
4158  * @param mark_id
4159  *   ID of MARK action, zero means default flow for META.
4160  * @param[out] error
4161  *   Perform verbose error reporting if not NULL.
4162  *
4163  * @return
4164  *   Associated resource on success, NULL otherwise and rte_errno is set.
4165  */
4166 static struct mlx5_flow_mreg_copy_resource *
4167 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4168                           struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         struct mlx5_hlist_entry *entry;
4172         struct mlx5_flow_cb_ctx ctx = {
4173                 .dev = dev,
4174                 .error = error,
4175         };
4176
4177         /* Check if already registered. */
4178         MLX5_ASSERT(priv->mreg_cp_tbl);
4179         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4180         if (!entry)
4181                 return NULL;
4182         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4183                             hlist_ent);
4184 }
4185
4186 void
4187 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
4188 {
4189         struct mlx5_flow_mreg_copy_resource *mcp_res =
4190                 container_of(entry, typeof(*mcp_res), hlist_ent);
4191         struct rte_eth_dev *dev = list->ctx;
4192         struct mlx5_priv *priv = dev->data->dev_private;
4193
4194         MLX5_ASSERT(mcp_res->rix_flow);
4195         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
4196         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4197 }
4198
4199 /**
4200  * Release flow in RX_CP_TBL.
4201  *
4202  * @param dev
4203  *   Pointer to Ethernet device.
4204  * @flow
4205  *   Parent flow for wich copying is provided.
4206  */
4207 static void
4208 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4209                           struct rte_flow *flow)
4210 {
4211         struct mlx5_flow_mreg_copy_resource *mcp_res;
4212         struct mlx5_priv *priv = dev->data->dev_private;
4213
4214         if (!flow->rix_mreg_copy)
4215                 return;
4216         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4217                                  flow->rix_mreg_copy);
4218         if (!mcp_res || !priv->mreg_cp_tbl)
4219                 return;
4220         MLX5_ASSERT(mcp_res->rix_flow);
4221         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4222         flow->rix_mreg_copy = 0;
4223 }
4224
4225 /**
4226  * Remove the default copy action from RX_CP_TBL.
4227  *
4228  * This functions is called in the mlx5_dev_start(). No thread safe
4229  * is guaranteed.
4230  *
4231  * @param dev
4232  *   Pointer to Ethernet device.
4233  */
4234 static void
4235 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4236 {
4237         struct mlx5_hlist_entry *entry;
4238         struct mlx5_priv *priv = dev->data->dev_private;
4239
4240         /* Check if default flow is registered. */
4241         if (!priv->mreg_cp_tbl)
4242                 return;
4243         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
4244                                   MLX5_DEFAULT_COPY_ID, NULL);
4245         if (!entry)
4246                 return;
4247         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4248 }
4249
4250 /**
4251  * Add the default copy action in in RX_CP_TBL.
4252  *
4253  * This functions is called in the mlx5_dev_start(). No thread safe
4254  * is guaranteed.
4255  *
4256  * @param dev
4257  *   Pointer to Ethernet device.
4258  * @param[out] error
4259  *   Perform verbose error reporting if not NULL.
4260  *
4261  * @return
4262  *   0 for success, negative value otherwise and rte_errno is set.
4263  */
4264 static int
4265 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4266                                   struct rte_flow_error *error)
4267 {
4268         struct mlx5_priv *priv = dev->data->dev_private;
4269         struct mlx5_flow_mreg_copy_resource *mcp_res;
4270
4271         /* Check whether extensive metadata feature is engaged. */
4272         if (!priv->config.dv_flow_en ||
4273             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4274             !mlx5_flow_ext_mreg_supported(dev) ||
4275             !priv->sh->dv_regc0_mask)
4276                 return 0;
4277         /*
4278          * Add default mreg copy flow may be called multiple time, but
4279          * only be called once in stop. Avoid register it twice.
4280          */
4281         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
4282                 return 0;
4283         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4284         if (!mcp_res)
4285                 return -rte_errno;
4286         return 0;
4287 }
4288
4289 /**
4290  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4291  *
4292  * All the flow having Q/RSS action should be split by
4293  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4294  * performs the following,
4295  *   - CQE->flow_tag := reg_c[1] (MARK)
4296  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4297  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4298  * but there should be a flow per each MARK ID set by MARK action.
4299  *
4300  * For the aforementioned reason, if there's a MARK action in flow's action
4301  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4302  * the MARK ID to CQE's flow_tag like,
4303  *   - If reg_c[1] is mark_id,
4304  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4305  *
4306  * For SET_META action which stores value in reg_c[0], as the destination is
4307  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4308  * MARK ID means the default flow. The default flow looks like,
4309  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4310  *
4311  * @param dev
4312  *   Pointer to Ethernet device.
4313  * @param flow
4314  *   Pointer to flow structure.
4315  * @param[in] actions
4316  *   Pointer to the list of actions.
4317  * @param[out] error
4318  *   Perform verbose error reporting if not NULL.
4319  *
4320  * @return
4321  *   0 on success, negative value otherwise and rte_errno is set.
4322  */
4323 static int
4324 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4325                             struct rte_flow *flow,
4326                             const struct rte_flow_action *actions,
4327                             struct rte_flow_error *error)
4328 {
4329         struct mlx5_priv *priv = dev->data->dev_private;
4330         struct mlx5_dev_config *config = &priv->config;
4331         struct mlx5_flow_mreg_copy_resource *mcp_res;
4332         const struct rte_flow_action_mark *mark;
4333
4334         /* Check whether extensive metadata feature is engaged. */
4335         if (!config->dv_flow_en ||
4336             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4337             !mlx5_flow_ext_mreg_supported(dev) ||
4338             !priv->sh->dv_regc0_mask)
4339                 return 0;
4340         /* Find MARK action. */
4341         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4342                 switch (actions->type) {
4343                 case RTE_FLOW_ACTION_TYPE_FLAG:
4344                         mcp_res = flow_mreg_add_copy_action
4345                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4346                         if (!mcp_res)
4347                                 return -rte_errno;
4348                         flow->rix_mreg_copy = mcp_res->idx;
4349                         return 0;
4350                 case RTE_FLOW_ACTION_TYPE_MARK:
4351                         mark = (const struct rte_flow_action_mark *)
4352                                 actions->conf;
4353                         mcp_res =
4354                                 flow_mreg_add_copy_action(dev, mark->id, error);
4355                         if (!mcp_res)
4356                                 return -rte_errno;
4357                         flow->rix_mreg_copy = mcp_res->idx;
4358                         return 0;
4359                 default:
4360                         break;
4361                 }
4362         }
4363         return 0;
4364 }
4365
4366 #define MLX5_MAX_SPLIT_ACTIONS 24
4367 #define MLX5_MAX_SPLIT_ITEMS 24
4368
4369 /**
4370  * Split the hairpin flow.
4371  * Since HW can't support encap and push-vlan on Rx, we move these
4372  * actions to Tx.
4373  * If the count action is after the encap then we also
4374  * move the count action. in this case the count will also measure
4375  * the outer bytes.
4376  *
4377  * @param dev
4378  *   Pointer to Ethernet device.
4379  * @param[in] actions
4380  *   Associated actions (list terminated by the END action).
4381  * @param[out] actions_rx
4382  *   Rx flow actions.
4383  * @param[out] actions_tx
4384  *   Tx flow actions..
4385  * @param[out] pattern_tx
4386  *   The pattern items for the Tx flow.
4387  * @param[out] flow_id
4388  *   The flow ID connected to this flow.
4389  *
4390  * @return
4391  *   0 on success.
4392  */
4393 static int
4394 flow_hairpin_split(struct rte_eth_dev *dev,
4395                    const struct rte_flow_action actions[],
4396                    struct rte_flow_action actions_rx[],
4397                    struct rte_flow_action actions_tx[],
4398                    struct rte_flow_item pattern_tx[],
4399                    uint32_t flow_id)
4400 {
4401         const struct rte_flow_action_raw_encap *raw_encap;
4402         const struct rte_flow_action_raw_decap *raw_decap;
4403         struct mlx5_rte_flow_action_set_tag *set_tag;
4404         struct rte_flow_action *tag_action;
4405         struct mlx5_rte_flow_item_tag *tag_item;
4406         struct rte_flow_item *item;
4407         char *addr;
4408         int encap = 0;
4409
4410         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4411                 switch (actions->type) {
4412                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4413                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4414                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4415                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4416                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4417                         rte_memcpy(actions_tx, actions,
4418                                sizeof(struct rte_flow_action));
4419                         actions_tx++;
4420                         break;
4421                 case RTE_FLOW_ACTION_TYPE_COUNT:
4422                         if (encap) {
4423                                 rte_memcpy(actions_tx, actions,
4424                                            sizeof(struct rte_flow_action));
4425                                 actions_tx++;
4426                         } else {
4427                                 rte_memcpy(actions_rx, actions,
4428                                            sizeof(struct rte_flow_action));
4429                                 actions_rx++;
4430                         }
4431                         break;
4432                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4433                         raw_encap = actions->conf;
4434                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4435                                 memcpy(actions_tx, actions,
4436                                        sizeof(struct rte_flow_action));
4437                                 actions_tx++;
4438                                 encap = 1;
4439                         } else {
4440                                 rte_memcpy(actions_rx, actions,
4441                                            sizeof(struct rte_flow_action));
4442                                 actions_rx++;
4443                         }
4444                         break;
4445                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4446                         raw_decap = actions->conf;
4447                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4448                                 memcpy(actions_tx, actions,
4449                                        sizeof(struct rte_flow_action));
4450                                 actions_tx++;
4451                         } else {
4452                                 rte_memcpy(actions_rx, actions,
4453                                            sizeof(struct rte_flow_action));
4454                                 actions_rx++;
4455                         }
4456                         break;
4457                 default:
4458                         rte_memcpy(actions_rx, actions,
4459                                    sizeof(struct rte_flow_action));
4460                         actions_rx++;
4461                         break;
4462                 }
4463         }
4464         /* Add set meta action and end action for the Rx flow. */
4465         tag_action = actions_rx;
4466         tag_action->type = (enum rte_flow_action_type)
4467                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4468         actions_rx++;
4469         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4470         actions_rx++;
4471         set_tag = (void *)actions_rx;
4472         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4473                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4474                 .data = flow_id,
4475         };
4476         MLX5_ASSERT(set_tag->id > REG_NON);
4477         tag_action->conf = set_tag;
4478         /* Create Tx item list. */
4479         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4480         addr = (void *)&pattern_tx[2];
4481         item = pattern_tx;
4482         item->type = (enum rte_flow_item_type)
4483                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4484         tag_item = (void *)addr;
4485         tag_item->data = flow_id;
4486         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4487         MLX5_ASSERT(set_tag->id > REG_NON);
4488         item->spec = tag_item;
4489         addr += sizeof(struct mlx5_rte_flow_item_tag);
4490         tag_item = (void *)addr;
4491         tag_item->data = UINT32_MAX;
4492         tag_item->id = UINT16_MAX;
4493         item->mask = tag_item;
4494         item->last = NULL;
4495         item++;
4496         item->type = RTE_FLOW_ITEM_TYPE_END;
4497         return 0;
4498 }
4499
4500 /**
4501  * The last stage of splitting chain, just creates the subflow
4502  * without any modification.
4503  *
4504  * @param[in] dev
4505  *   Pointer to Ethernet device.
4506  * @param[in] flow
4507  *   Parent flow structure pointer.
4508  * @param[in, out] sub_flow
4509  *   Pointer to return the created subflow, may be NULL.
4510  * @param[in] attr
4511  *   Flow rule attributes.
4512  * @param[in] items
4513  *   Pattern specification (list terminated by the END pattern item).
4514  * @param[in] actions
4515  *   Associated actions (list terminated by the END action).
4516  * @param[in] flow_split_info
4517  *   Pointer to flow split info structure.
4518  * @param[out] error
4519  *   Perform verbose error reporting if not NULL.
4520  * @return
4521  *   0 on success, negative value otherwise
4522  */
4523 static int
4524 flow_create_split_inner(struct rte_eth_dev *dev,
4525                         struct rte_flow *flow,
4526                         struct mlx5_flow **sub_flow,
4527                         const struct rte_flow_attr *attr,
4528                         const struct rte_flow_item items[],
4529                         const struct rte_flow_action actions[],
4530                         struct mlx5_flow_split_info *flow_split_info,
4531                         struct rte_flow_error *error)
4532 {
4533         struct mlx5_flow *dev_flow;
4534
4535         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4536                                     flow_split_info->flow_idx, error);
4537         if (!dev_flow)
4538                 return -rte_errno;
4539         dev_flow->flow = flow;
4540         dev_flow->external = flow_split_info->external;
4541         dev_flow->skip_scale = flow_split_info->skip_scale;
4542         /* Subflow object was created, we must include one in the list. */
4543         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4544                       dev_flow->handle, next);
4545         /*
4546          * If dev_flow is as one of the suffix flow, some actions in suffix
4547          * flow may need some user defined item layer flags, and pass the
4548          * Metadate rxq mark flag to suffix flow as well.
4549          */
4550         if (flow_split_info->prefix_layers)
4551                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4552         if (flow_split_info->prefix_mark)
4553                 dev_flow->handle->mark = 1;
4554         if (sub_flow)
4555                 *sub_flow = dev_flow;
4556 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4557         dev_flow->dv.table_id = flow_split_info->table_id;
4558 #endif
4559         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4560 }
4561
4562 /**
4563  * Get the sub policy of a meter.
4564  *
4565  * @param[in] dev
4566  *   Pointer to Ethernet device.
4567  * @param[in] flow
4568  *   Parent flow structure pointer.
4569  * @param[in] policy_id;
4570  *   Meter Policy id.
4571  * @param[in] attr
4572  *   Flow rule attributes.
4573  * @param[in] items
4574  *   Pattern specification (list terminated by the END pattern item).
4575  * @param[out] error
4576  *   Perform verbose error reporting if not NULL.
4577  *
4578  * @return
4579  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4580  */
4581 static struct mlx5_flow_meter_sub_policy *
4582 get_meter_sub_policy(struct rte_eth_dev *dev,
4583                      struct rte_flow *flow,
4584                      uint32_t policy_id,
4585                      const struct rte_flow_attr *attr,
4586                      const struct rte_flow_item items[],
4587                      struct rte_flow_error *error)
4588 {
4589         struct mlx5_flow_meter_policy *policy;
4590         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4591
4592         policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL);
4593         if (!policy) {
4594                 rte_flow_error_set(error, EINVAL,
4595                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4596                                    "Failed to find Meter Policy.");
4597                 goto exit;
4598         }
4599         if (policy->is_rss ||
4600                 (policy->is_queue &&
4601         !policy->sub_policys[MLX5_MTR_DOMAIN_INGRESS][0]->rix_hrxq[0])) {
4602                 struct mlx5_flow_workspace *wks =
4603                                 mlx5_flow_get_thread_workspace();
4604                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4605                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4606                 uint32_t i;
4607
4608                 MLX5_ASSERT(wks);
4609                 /**
4610                  * This is a tmp dev_flow,
4611                  * no need to register any matcher for it in translate.
4612                  */
4613                 wks->skip_matcher_reg = 1;
4614                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4615                         struct mlx5_flow dev_flow = {0};
4616                         struct mlx5_flow_handle dev_handle = { {0} };
4617
4618                         if (policy->is_rss) {
4619                                 const void *rss_act =
4620                                         policy->act_cnt[i].rss->conf;
4621                                 struct rte_flow_action rss_actions[2] = {
4622                                         [0] = {
4623                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4624                                         .conf = rss_act
4625                                         },
4626                                         [1] = {
4627                                         .type = RTE_FLOW_ACTION_TYPE_END,
4628                                         .conf = NULL
4629                                         }
4630                                 };
4631
4632                                 dev_flow.handle = &dev_handle;
4633                                 dev_flow.ingress = attr->ingress;
4634                                 dev_flow.flow = flow;
4635                                 dev_flow.external = 0;
4636 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4637                                 dev_flow.dv.transfer = attr->transfer;
4638 #endif
4639                                 /**
4640                                  * Translate RSS action to get rss hash fields.
4641                                  */
4642                                 if (flow_drv_translate(dev, &dev_flow, attr,
4643                                                 items, rss_actions, error))
4644                                         goto exit;
4645                                 rss_desc_v[i] = wks->rss_desc;
4646                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4647                                 rss_desc_v[i].hash_fields =
4648                                                 dev_flow.hash_fields;
4649                                 rss_desc_v[i].queue_num =
4650                                                 rss_desc_v[i].hash_fields ?
4651                                                 rss_desc_v[i].queue_num : 1;
4652                                 rss_desc_v[i].tunnel =
4653                                         !!(dev_flow.handle->layers &
4654                                         MLX5_FLOW_LAYER_TUNNEL);
4655                         } else {
4656                                 /* This is queue action. */
4657                                 rss_desc_v[i] = wks->rss_desc;
4658                                 rss_desc_v[i].key_len = 0;
4659                                 rss_desc_v[i].hash_fields = 0;
4660                                 rss_desc_v[i].queue =
4661                                         &policy->act_cnt[i].queue;
4662                                 rss_desc_v[i].queue_num = 1;
4663                         }
4664                         rss_desc[i] = &rss_desc_v[i];
4665                 }
4666                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4667                                                 flow, policy, rss_desc);
4668         } else {
4669                 enum mlx5_meter_domain mtr_domain =
4670                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4671                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4672                                         MLX5_MTR_DOMAIN_INGRESS;
4673                 sub_policy = policy->sub_policys[mtr_domain][0];
4674         }
4675         if (!sub_policy) {
4676                 rte_flow_error_set(error, EINVAL,
4677                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4678                         "Failed to get meter sub-policy.");
4679                 goto exit;
4680         }
4681 exit:
4682         return sub_policy;
4683 }
4684
4685 /**
4686  * Split the meter flow.
4687  *
4688  * As meter flow will split to three sub flow, other than meter
4689  * action, the other actions make sense to only meter accepts
4690  * the packet. If it need to be dropped, no other additional
4691  * actions should be take.
4692  *
4693  * One kind of special action which decapsulates the L3 tunnel
4694  * header will be in the prefix sub flow, as not to take the
4695  * L3 tunnel header into account.
4696  *
4697  * @param[in] dev
4698  *   Pointer to Ethernet device.
4699  * @param[in] flow
4700  *   Parent flow structure pointer.
4701  * @param[in] fm
4702  *   Pointer to flow meter structure.
4703  * @param[in] attr
4704  *   Flow rule attributes.
4705  * @param[in] items
4706  *   Pattern specification (list terminated by the END pattern item).
4707  * @param[out] sfx_items
4708  *   Suffix flow match items (list terminated by the END pattern item).
4709  * @param[in] actions
4710  *   Associated actions (list terminated by the END action).
4711  * @param[out] actions_sfx
4712  *   Suffix flow actions.
4713  * @param[out] actions_pre
4714  *   Prefix flow actions.
4715  * @param[out] mtr_flow_id
4716  *   Pointer to meter flow id.
4717  * @param[out] error
4718  *   Perform verbose error reporting if not NULL.
4719  *
4720  * @return
4721  *   0 on success, a negative errno value otherwise and rte_errno is set.
4722  */
4723 static int
4724 flow_meter_split_prep(struct rte_eth_dev *dev,
4725                       struct rte_flow *flow,
4726                       struct mlx5_flow_meter_info *fm,
4727                       const struct rte_flow_attr *attr,
4728                       const struct rte_flow_item items[],
4729                       struct rte_flow_item sfx_items[],
4730                       const struct rte_flow_action actions[],
4731                       struct rte_flow_action actions_sfx[],
4732                       struct rte_flow_action actions_pre[],
4733                       uint32_t *mtr_flow_id,
4734                       struct rte_flow_error *error)
4735 {
4736         struct mlx5_priv *priv = dev->data->dev_private;
4737         struct rte_flow_action *tag_action = NULL;
4738         struct rte_flow_item *tag_item;
4739         struct mlx5_rte_flow_action_set_tag *set_tag;
4740         const struct rte_flow_action_raw_encap *raw_encap;
4741         const struct rte_flow_action_raw_decap *raw_decap;
4742         struct mlx5_rte_flow_item_tag *tag_item_spec;
4743         struct mlx5_rte_flow_item_tag *tag_item_mask;
4744         uint32_t tag_id = 0;
4745         struct rte_flow_item *vlan_item_dst = NULL;
4746         const struct rte_flow_item *vlan_item_src = NULL;
4747         struct rte_flow_action *hw_mtr_action;
4748         struct rte_flow_action *action_pre_head = NULL;
4749         int32_t flow_src_port = priv->representor_id;
4750         bool mtr_first;
4751         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4752         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4753                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4754         uint32_t flow_id = 0;
4755         uint32_t flow_id_reversed = 0;
4756         uint8_t flow_id_bits = 0;
4757         int shift;
4758
4759         /* Prepare the suffix subflow items. */
4760         tag_item = sfx_items++;
4761         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4762                 struct mlx5_priv *port_priv;
4763                 const struct rte_flow_item_port_id *pid_v;
4764                 int item_type = items->type;
4765
4766                 switch (item_type) {
4767                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4768                         pid_v = items->spec;
4769                         MLX5_ASSERT(pid_v);
4770                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
4771                         if (!port_priv)
4772                                 return rte_flow_error_set(error,
4773                                                 rte_errno,
4774                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
4775                                                 pid_v,
4776                                                 "Failed to get port info.");
4777                         flow_src_port = port_priv->representor_id;
4778                         memcpy(sfx_items, items, sizeof(*sfx_items));
4779                         sfx_items++;
4780                         break;
4781                 case RTE_FLOW_ITEM_TYPE_VLAN:
4782                         /* Determine if copy vlan item below. */
4783                         vlan_item_src = items;
4784                         vlan_item_dst = sfx_items++;
4785                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
4786                         break;
4787                 default:
4788                         break;
4789                 }
4790         }
4791         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4792         sfx_items++;
4793         mtr_first = priv->sh->meter_aso_en &&
4794                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
4795         /* For ASO meter, meter must be before tag in TX direction. */
4796         if (mtr_first) {
4797                 action_pre_head = actions_pre++;
4798                 /* Leave space for tag action. */
4799                 tag_action = actions_pre++;
4800         }
4801         /* Prepare the actions for prefix and suffix flow. */
4802         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4803                 struct rte_flow_action *action_cur = NULL;
4804
4805                 switch (actions->type) {
4806                 case RTE_FLOW_ACTION_TYPE_METER:
4807                         if (mtr_first) {
4808                                 action_cur = action_pre_head;
4809                         } else {
4810                                 /* Leave space for tag action. */
4811                                 tag_action = actions_pre++;
4812                                 action_cur = actions_pre++;
4813                         }
4814                         break;
4815                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4816                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4817                         action_cur = actions_pre++;
4818                         break;
4819                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4820                         raw_encap = actions->conf;
4821                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4822                                 action_cur = actions_pre++;
4823                         break;
4824                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4825                         raw_decap = actions->conf;
4826                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4827                                 action_cur = actions_pre++;
4828                         break;
4829                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4830                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4831                         if (vlan_item_dst && vlan_item_src) {
4832                                 memcpy(vlan_item_dst, vlan_item_src,
4833                                         sizeof(*vlan_item_dst));
4834                                 /*
4835                                  * Convert to internal match item, it is used
4836                                  * for vlan push and set vid.
4837                                  */
4838                                 vlan_item_dst->type = (enum rte_flow_item_type)
4839                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4840                         }
4841                         break;
4842                 default:
4843                         break;
4844                 }
4845                 if (!action_cur)
4846                         action_cur = (fm->def_policy) ?
4847                                         actions_sfx++ : actions_pre++;
4848                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4849         }
4850         /* Add end action to the actions. */
4851         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4852         if (priv->sh->meter_aso_en) {
4853                 /**
4854                  * For ASO meter, need to add an extra jump action explicitly,
4855                  * to jump from meter to policer table.
4856                  */
4857                 struct mlx5_flow_meter_sub_policy *sub_policy;
4858                 struct mlx5_flow_tbl_data_entry *tbl_data;
4859
4860                 if (!fm->def_policy) {
4861                         sub_policy = get_meter_sub_policy(dev, flow,
4862                                                           fm->policy_id, attr,
4863                                                           items, error);
4864                         if (!sub_policy)
4865                                 return -rte_errno;
4866                 } else {
4867                         enum mlx5_meter_domain mtr_domain =
4868                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4869                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4870                                         MLX5_MTR_DOMAIN_INGRESS;
4871
4872                         sub_policy =
4873                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
4874                 }
4875                 tbl_data = container_of(sub_policy->tbl_rsc,
4876                                         struct mlx5_flow_tbl_data_entry, tbl);
4877                 hw_mtr_action = actions_pre++;
4878                 hw_mtr_action->type = (enum rte_flow_action_type)
4879                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
4880                 hw_mtr_action->conf = tbl_data->jump.action;
4881         }
4882         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4883         actions_pre++;
4884         if (!tag_action)
4885                 return rte_flow_error_set(error, ENOMEM,
4886                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4887                                         "No tag action space.");
4888         if (!mtr_flow_id) {
4889                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
4890                 goto exit;
4891         }
4892         /* Only default-policy Meter creates mtr flow id. */
4893         if (fm->def_policy) {
4894                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
4895                 if (!tag_id)
4896                         return rte_flow_error_set(error, ENOMEM,
4897                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4898                                         "Failed to allocate meter flow id.");
4899                 flow_id = tag_id - 1;
4900                 flow_id_bits = (!flow_id) ? 1 :
4901                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
4902                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
4903                     mtr_reg_bits) {
4904                         mlx5_ipool_free(fm->flow_ipool, tag_id);
4905                         return rte_flow_error_set(error, EINVAL,
4906                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4907                                         "Meter flow id exceeds max limit.");
4908                 }
4909                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
4910                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
4911         }
4912         /* Build tag actions and items for meter_id/meter flow_id. */
4913         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
4914         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4915         tag_item_mask = tag_item_spec + 1;
4916         /* Both flow_id and meter_id share the same register. */
4917         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4918                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
4919                                                             0, error),
4920                 .offset = mtr_id_offset,
4921                 .length = mtr_reg_bits,
4922                 .data = flow->meter,
4923         };
4924         /*
4925          * The color Reg bits used by flow_id are growing from
4926          * msb to lsb, so must do bit reverse for flow_id val in RegC.
4927          */
4928         for (shift = 0; shift < flow_id_bits; shift++)
4929                 flow_id_reversed = (flow_id_reversed << 1) |
4930                                 ((flow_id >> shift) & 0x1);
4931         set_tag->data |=
4932                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
4933         tag_item_spec->id = set_tag->id;
4934         tag_item_spec->data = set_tag->data << mtr_id_offset;
4935         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
4936         tag_action->type = (enum rte_flow_action_type)
4937                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4938         tag_action->conf = set_tag;
4939         tag_item->type = (enum rte_flow_item_type)
4940                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4941         tag_item->spec = tag_item_spec;
4942         tag_item->last = NULL;
4943         tag_item->mask = tag_item_mask;
4944 exit:
4945         if (mtr_flow_id)
4946                 *mtr_flow_id = tag_id;
4947         return 0;
4948 }
4949
4950 /**
4951  * Split action list having QUEUE/RSS for metadata register copy.
4952  *
4953  * Once Q/RSS action is detected in user's action list, the flow action
4954  * should be split in order to copy metadata registers, which will happen in
4955  * RX_CP_TBL like,
4956  *   - CQE->flow_tag := reg_c[1] (MARK)
4957  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4958  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4959  * This is because the last action of each flow must be a terminal action
4960  * (QUEUE, RSS or DROP).
4961  *
4962  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4963  * stored and kept in the mlx5_flow structure per each sub_flow.
4964  *
4965  * The Q/RSS action is replaced with,
4966  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4967  * And the following JUMP action is added at the end,
4968  *   - JUMP, to RX_CP_TBL.
4969  *
4970  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4971  * flow_create_split_metadata() routine. The flow will look like,
4972  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4973  *
4974  * @param dev
4975  *   Pointer to Ethernet device.
4976  * @param[out] split_actions
4977  *   Pointer to store split actions to jump to CP_TBL.
4978  * @param[in] actions
4979  *   Pointer to the list of original flow actions.
4980  * @param[in] qrss
4981  *   Pointer to the Q/RSS action.
4982  * @param[in] actions_n
4983  *   Number of original actions.
4984  * @param[out] error
4985  *   Perform verbose error reporting if not NULL.
4986  *
4987  * @return
4988  *   non-zero unique flow_id on success, otherwise 0 and
4989  *   error/rte_error are set.
4990  */
4991 static uint32_t
4992 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4993                           struct rte_flow_action *split_actions,
4994                           const struct rte_flow_action *actions,
4995                           const struct rte_flow_action *qrss,
4996                           int actions_n, struct rte_flow_error *error)
4997 {
4998         struct mlx5_priv *priv = dev->data->dev_private;
4999         struct mlx5_rte_flow_action_set_tag *set_tag;
5000         struct rte_flow_action_jump *jump;
5001         const int qrss_idx = qrss - actions;
5002         uint32_t flow_id = 0;
5003         int ret = 0;
5004
5005         /*
5006          * Given actions will be split
5007          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5008          * - Add jump to mreg CP_TBL.
5009          * As a result, there will be one more action.
5010          */
5011         ++actions_n;
5012         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5013         set_tag = (void *)(split_actions + actions_n);
5014         /*
5015          * If tag action is not set to void(it means we are not the meter
5016          * suffix flow), add the tag action. Since meter suffix flow already
5017          * has the tag added.
5018          */
5019         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
5020                 /*
5021                  * Allocate the new subflow ID. This one is unique within
5022                  * device and not shared with representors. Otherwise,
5023                  * we would have to resolve multi-thread access synch
5024                  * issue. Each flow on the shared device is appended
5025                  * with source vport identifier, so the resulting
5026                  * flows will be unique in the shared (by master and
5027                  * representors) domain even if they have coinciding
5028                  * IDs.
5029                  */
5030                 mlx5_ipool_malloc(priv->sh->ipool
5031                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5032                 if (!flow_id)
5033                         return rte_flow_error_set(error, ENOMEM,
5034                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5035                                                   NULL, "can't allocate id "
5036                                                   "for split Q/RSS subflow");
5037                 /* Internal SET_TAG action to set flow ID. */
5038                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5039                         .data = flow_id,
5040                 };
5041                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5042                 if (ret < 0)
5043                         return ret;
5044                 set_tag->id = ret;
5045                 /* Construct new actions array. */
5046                 /* Replace QUEUE/RSS action. */
5047                 split_actions[qrss_idx] = (struct rte_flow_action){
5048                         .type = (enum rte_flow_action_type)
5049                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5050                         .conf = set_tag,
5051                 };
5052         }
5053         /* JUMP action to jump to mreg copy table (CP_TBL). */
5054         jump = (void *)(set_tag + 1);
5055         *jump = (struct rte_flow_action_jump){
5056                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5057         };
5058         split_actions[actions_n - 2] = (struct rte_flow_action){
5059                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5060                 .conf = jump,
5061         };
5062         split_actions[actions_n - 1] = (struct rte_flow_action){
5063                 .type = RTE_FLOW_ACTION_TYPE_END,
5064         };
5065         return flow_id;
5066 }
5067
5068 /**
5069  * Extend the given action list for Tx metadata copy.
5070  *
5071  * Copy the given action list to the ext_actions and add flow metadata register
5072  * copy action in order to copy reg_a set by WQE to reg_c[0].
5073  *
5074  * @param[out] ext_actions
5075  *   Pointer to the extended action list.
5076  * @param[in] actions
5077  *   Pointer to the list of actions.
5078  * @param[in] actions_n
5079  *   Number of actions in the list.
5080  * @param[out] error
5081  *   Perform verbose error reporting if not NULL.
5082  * @param[in] encap_idx
5083  *   The encap action inndex.
5084  *
5085  * @return
5086  *   0 on success, negative value otherwise
5087  */
5088 static int
5089 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5090                        struct rte_flow_action *ext_actions,
5091                        const struct rte_flow_action *actions,
5092                        int actions_n, struct rte_flow_error *error,
5093                        int encap_idx)
5094 {
5095         struct mlx5_flow_action_copy_mreg *cp_mreg =
5096                 (struct mlx5_flow_action_copy_mreg *)
5097                         (ext_actions + actions_n + 1);
5098         int ret;
5099
5100         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5101         if (ret < 0)
5102                 return ret;
5103         cp_mreg->dst = ret;
5104         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5105         if (ret < 0)
5106                 return ret;
5107         cp_mreg->src = ret;
5108         if (encap_idx != 0)
5109                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5110         if (encap_idx == actions_n - 1) {
5111                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5112                         .type = (enum rte_flow_action_type)
5113                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5114                         .conf = cp_mreg,
5115                 };
5116                 ext_actions[actions_n] = (struct rte_flow_action){
5117                         .type = RTE_FLOW_ACTION_TYPE_END,
5118                 };
5119         } else {
5120                 ext_actions[encap_idx] = (struct rte_flow_action){
5121                         .type = (enum rte_flow_action_type)
5122                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5123                         .conf = cp_mreg,
5124                 };
5125                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5126                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5127         }
5128         return 0;
5129 }
5130
5131 /**
5132  * Check the match action from the action list.
5133  *
5134  * @param[in] actions
5135  *   Pointer to the list of actions.
5136  * @param[in] attr
5137  *   Flow rule attributes.
5138  * @param[in] action
5139  *   The action to be check if exist.
5140  * @param[out] match_action_pos
5141  *   Pointer to the position of the matched action if exists, otherwise is -1.
5142  * @param[out] qrss_action_pos
5143  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5144  * @param[out] modify_after_mirror
5145  *   Pointer to the flag of modify action after FDB mirroring.
5146  *
5147  * @return
5148  *   > 0 the total number of actions.
5149  *   0 if not found match action in action list.
5150  */
5151 static int
5152 flow_check_match_action(const struct rte_flow_action actions[],
5153                         const struct rte_flow_attr *attr,
5154                         enum rte_flow_action_type action,
5155                         int *match_action_pos, int *qrss_action_pos,
5156                         int *modify_after_mirror)
5157 {
5158         const struct rte_flow_action_sample *sample;
5159         int actions_n = 0;
5160         uint32_t ratio = 0;
5161         int sub_type = 0;
5162         int flag = 0;
5163         int fdb_mirror = 0;
5164
5165         *match_action_pos = -1;
5166         *qrss_action_pos = -1;
5167         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5168                 if (actions->type == action) {
5169                         flag = 1;
5170                         *match_action_pos = actions_n;
5171                 }
5172                 switch (actions->type) {
5173                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5174                 case RTE_FLOW_ACTION_TYPE_RSS:
5175                         *qrss_action_pos = actions_n;
5176                         break;
5177                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5178                         sample = actions->conf;
5179                         ratio = sample->ratio;
5180                         sub_type = ((const struct rte_flow_action *)
5181                                         (sample->actions))->type;
5182                         if (ratio == 1 && attr->transfer)
5183                                 fdb_mirror = 1;
5184                         break;
5185                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5186                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5187                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5188                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5189                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5190                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5191                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5192                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5193                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5194                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5195                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5196                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5197                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5198                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5199                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5200                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5201                 case RTE_FLOW_ACTION_TYPE_FLAG:
5202                 case RTE_FLOW_ACTION_TYPE_MARK:
5203                 case RTE_FLOW_ACTION_TYPE_SET_META:
5204                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5205                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5206                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5207                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5208                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5209                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5210                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5211                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5212                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5213                 case RTE_FLOW_ACTION_TYPE_METER:
5214                         if (fdb_mirror)
5215                                 *modify_after_mirror = 1;
5216                         break;
5217                 default:
5218                         break;
5219                 }
5220                 actions_n++;
5221         }
5222         if (flag && fdb_mirror && !*modify_after_mirror) {
5223                 /* FDB mirroring uses the destination array to implement
5224                  * instead of FLOW_SAMPLER object.
5225                  */
5226                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5227                         flag = 0;
5228         }
5229         /* Count RTE_FLOW_ACTION_TYPE_END. */
5230         return flag ? actions_n + 1 : 0;
5231 }
5232
5233 #define SAMPLE_SUFFIX_ITEM 2
5234
5235 /**
5236  * Split the sample flow.
5237  *
5238  * As sample flow will split to two sub flow, sample flow with
5239  * sample action, the other actions will move to new suffix flow.
5240  *
5241  * Also add unique tag id with tag action in the sample flow,
5242  * the same tag id will be as match in the suffix flow.
5243  *
5244  * @param dev
5245  *   Pointer to Ethernet device.
5246  * @param[in] add_tag
5247  *   Add extra tag action flag.
5248  * @param[out] sfx_items
5249  *   Suffix flow match items (list terminated by the END pattern item).
5250  * @param[in] actions
5251  *   Associated actions (list terminated by the END action).
5252  * @param[out] actions_sfx
5253  *   Suffix flow actions.
5254  * @param[out] actions_pre
5255  *   Prefix flow actions.
5256  * @param[in] actions_n
5257  *  The total number of actions.
5258  * @param[in] sample_action_pos
5259  *   The sample action position.
5260  * @param[in] qrss_action_pos
5261  *   The Queue/RSS action position.
5262  * @param[in] jump_table
5263  *   Add extra jump action flag.
5264  * @param[out] error
5265  *   Perform verbose error reporting if not NULL.
5266  *
5267  * @return
5268  *   0 on success, or unique flow_id, a negative errno value
5269  *   otherwise and rte_errno is set.
5270  */
5271 static int
5272 flow_sample_split_prep(struct rte_eth_dev *dev,
5273                        int add_tag,
5274                        struct rte_flow_item sfx_items[],
5275                        const struct rte_flow_action actions[],
5276                        struct rte_flow_action actions_sfx[],
5277                        struct rte_flow_action actions_pre[],
5278                        int actions_n,
5279                        int sample_action_pos,
5280                        int qrss_action_pos,
5281                        int jump_table,
5282                        struct rte_flow_error *error)
5283 {
5284         struct mlx5_priv *priv = dev->data->dev_private;
5285         struct mlx5_rte_flow_action_set_tag *set_tag;
5286         struct mlx5_rte_flow_item_tag *tag_spec;
5287         struct mlx5_rte_flow_item_tag *tag_mask;
5288         struct rte_flow_action_jump *jump_action;
5289         uint32_t tag_id = 0;
5290         int index;
5291         int append_index = 0;
5292         int ret;
5293
5294         if (sample_action_pos < 0)
5295                 return rte_flow_error_set(error, EINVAL,
5296                                           RTE_FLOW_ERROR_TYPE_ACTION,
5297                                           NULL, "invalid position of sample "
5298                                           "action in list");
5299         /* Prepare the actions for prefix and suffix flow. */
5300         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5301                 index = qrss_action_pos;
5302                 /* Put the preceding the Queue/RSS action into prefix flow. */
5303                 if (index != 0)
5304                         memcpy(actions_pre, actions,
5305                                sizeof(struct rte_flow_action) * index);
5306                 /* Put others preceding the sample action into prefix flow. */
5307                 if (sample_action_pos > index + 1)
5308                         memcpy(actions_pre + index, actions + index + 1,
5309                                sizeof(struct rte_flow_action) *
5310                                (sample_action_pos - index - 1));
5311                 index = sample_action_pos - 1;
5312                 /* Put Queue/RSS action into Suffix flow. */
5313                 memcpy(actions_sfx, actions + qrss_action_pos,
5314                        sizeof(struct rte_flow_action));
5315                 actions_sfx++;
5316         } else {
5317                 index = sample_action_pos;
5318                 if (index != 0)
5319                         memcpy(actions_pre, actions,
5320                                sizeof(struct rte_flow_action) * index);
5321         }
5322         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5323          * For CX6DX and above, metadata registers Cx preserve their value,
5324          * add an extra tag action for NIC-RX and E-Switch Domain.
5325          */
5326         if (add_tag) {
5327                 /* Prepare the prefix tag action. */
5328                 append_index++;
5329                 set_tag = (void *)(actions_pre + actions_n + append_index);
5330                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5331                 if (ret < 0)
5332                         return ret;
5333                 mlx5_ipool_malloc(priv->sh->ipool
5334                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5335                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5336                         .id = ret,
5337                         .data = tag_id,
5338                 };
5339                 /* Prepare the suffix subflow items. */
5340                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5341                 tag_spec->data = tag_id;
5342                 tag_spec->id = set_tag->id;
5343                 tag_mask = tag_spec + 1;
5344                 tag_mask->data = UINT32_MAX;
5345                 sfx_items[0] = (struct rte_flow_item){
5346                         .type = (enum rte_flow_item_type)
5347                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5348                         .spec = tag_spec,
5349                         .last = NULL,
5350                         .mask = tag_mask,
5351                 };
5352                 sfx_items[1] = (struct rte_flow_item){
5353                         .type = (enum rte_flow_item_type)
5354                                 RTE_FLOW_ITEM_TYPE_END,
5355                 };
5356                 /* Prepare the tag action in prefix subflow. */
5357                 actions_pre[index++] =
5358                         (struct rte_flow_action){
5359                         .type = (enum rte_flow_action_type)
5360                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5361                         .conf = set_tag,
5362                 };
5363         }
5364         memcpy(actions_pre + index, actions + sample_action_pos,
5365                sizeof(struct rte_flow_action));
5366         index += 1;
5367         /* For the modify action after the sample action in E-Switch mirroring,
5368          * Add the extra jump action in prefix subflow and jump into the next
5369          * table, then do the modify action in the new table.
5370          */
5371         if (jump_table) {
5372                 /* Prepare the prefix jump action. */
5373                 append_index++;
5374                 jump_action = (void *)(actions_pre + actions_n + append_index);
5375                 jump_action->group = jump_table;
5376                 actions_pre[index++] =
5377                         (struct rte_flow_action){
5378                         .type = (enum rte_flow_action_type)
5379                                 RTE_FLOW_ACTION_TYPE_JUMP,
5380                         .conf = jump_action,
5381                 };
5382         }
5383         actions_pre[index] = (struct rte_flow_action){
5384                 .type = (enum rte_flow_action_type)
5385                         RTE_FLOW_ACTION_TYPE_END,
5386         };
5387         /* Put the actions after sample into Suffix flow. */
5388         memcpy(actions_sfx, actions + sample_action_pos + 1,
5389                sizeof(struct rte_flow_action) *
5390                (actions_n - sample_action_pos - 1));
5391         return tag_id;
5392 }
5393
5394 /**
5395  * The splitting for metadata feature.
5396  *
5397  * - Q/RSS action on NIC Rx should be split in order to pass by
5398  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5399  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5400  *
5401  * - All the actions on NIC Tx should have a mreg copy action to
5402  *   copy reg_a from WQE to reg_c[0].
5403  *
5404  * @param dev
5405  *   Pointer to Ethernet device.
5406  * @param[in] flow
5407  *   Parent flow structure pointer.
5408  * @param[in] attr
5409  *   Flow rule attributes.
5410  * @param[in] items
5411  *   Pattern specification (list terminated by the END pattern item).
5412  * @param[in] actions
5413  *   Associated actions (list terminated by the END action).
5414  * @param[in] flow_split_info
5415  *   Pointer to flow split info structure.
5416  * @param[out] error
5417  *   Perform verbose error reporting if not NULL.
5418  * @return
5419  *   0 on success, negative value otherwise
5420  */
5421 static int
5422 flow_create_split_metadata(struct rte_eth_dev *dev,
5423                            struct rte_flow *flow,
5424                            const struct rte_flow_attr *attr,
5425                            const struct rte_flow_item items[],
5426                            const struct rte_flow_action actions[],
5427                            struct mlx5_flow_split_info *flow_split_info,
5428                            struct rte_flow_error *error)
5429 {
5430         struct mlx5_priv *priv = dev->data->dev_private;
5431         struct mlx5_dev_config *config = &priv->config;
5432         const struct rte_flow_action *qrss = NULL;
5433         struct rte_flow_action *ext_actions = NULL;
5434         struct mlx5_flow *dev_flow = NULL;
5435         uint32_t qrss_id = 0;
5436         int mtr_sfx = 0;
5437         size_t act_size;
5438         int actions_n;
5439         int encap_idx;
5440         int ret;
5441
5442         /* Check whether extensive metadata feature is engaged. */
5443         if (!config->dv_flow_en ||
5444             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5445             !mlx5_flow_ext_mreg_supported(dev))
5446                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5447                                                actions, flow_split_info, error);
5448         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5449                                                            &encap_idx);
5450         if (qrss) {
5451                 /* Exclude hairpin flows from splitting. */
5452                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5453                         const struct rte_flow_action_queue *queue;
5454
5455                         queue = qrss->conf;
5456                         if (mlx5_rxq_get_type(dev, queue->index) ==
5457                             MLX5_RXQ_TYPE_HAIRPIN)
5458                                 qrss = NULL;
5459                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5460                         const struct rte_flow_action_rss *rss;
5461
5462                         rss = qrss->conf;
5463                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5464                             MLX5_RXQ_TYPE_HAIRPIN)
5465                                 qrss = NULL;
5466                 }
5467         }
5468         if (qrss) {
5469                 /* Check if it is in meter suffix table. */
5470                 mtr_sfx = attr->group == (attr->transfer ?
5471                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5472                           MLX5_FLOW_TABLE_LEVEL_METER);
5473                 /*
5474                  * Q/RSS action on NIC Rx should be split in order to pass by
5475                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5476                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5477                  */
5478                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5479                            sizeof(struct rte_flow_action_set_tag) +
5480                            sizeof(struct rte_flow_action_jump);
5481                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5482                                           SOCKET_ID_ANY);
5483                 if (!ext_actions)
5484                         return rte_flow_error_set(error, ENOMEM,
5485                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5486                                                   NULL, "no memory to split "
5487                                                   "metadata flow");
5488                 /*
5489                  * If we are the suffix flow of meter, tag already exist.
5490                  * Set the tag action to void.
5491                  */
5492                 if (mtr_sfx)
5493                         ext_actions[qrss - actions].type =
5494                                                 RTE_FLOW_ACTION_TYPE_VOID;
5495                 else
5496                         ext_actions[qrss - actions].type =
5497                                                 (enum rte_flow_action_type)
5498                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5499                 /*
5500                  * Create the new actions list with removed Q/RSS action
5501                  * and appended set tag and jump to register copy table
5502                  * (RX_CP_TBL). We should preallocate unique tag ID here
5503                  * in advance, because it is needed for set tag action.
5504                  */
5505                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5506                                                     qrss, actions_n, error);
5507                 if (!mtr_sfx && !qrss_id) {
5508                         ret = -rte_errno;
5509                         goto exit;
5510                 }
5511         } else if (attr->egress && !attr->transfer) {
5512                 /*
5513                  * All the actions on NIC Tx should have a metadata register
5514                  * copy action to copy reg_a from WQE to reg_c[meta]
5515                  */
5516                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5517                            sizeof(struct mlx5_flow_action_copy_mreg);
5518                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5519                                           SOCKET_ID_ANY);
5520                 if (!ext_actions)
5521                         return rte_flow_error_set(error, ENOMEM,
5522                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5523                                                   NULL, "no memory to split "
5524                                                   "metadata flow");
5525                 /* Create the action list appended with copy register. */
5526                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5527                                              actions_n, error, encap_idx);
5528                 if (ret < 0)
5529                         goto exit;
5530         }
5531         /* Add the unmodified original or prefix subflow. */
5532         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5533                                       items, ext_actions ? ext_actions :
5534                                       actions, flow_split_info, error);
5535         if (ret < 0)
5536                 goto exit;
5537         MLX5_ASSERT(dev_flow);
5538         if (qrss) {
5539                 const struct rte_flow_attr q_attr = {
5540                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5541                         .ingress = 1,
5542                 };
5543                 /* Internal PMD action to set register. */
5544                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5545                         .data = qrss_id,
5546                         .id = REG_NON,
5547                 };
5548                 struct rte_flow_item q_items[] = {
5549                         {
5550                                 .type = (enum rte_flow_item_type)
5551                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5552                                 .spec = &q_tag_spec,
5553                                 .last = NULL,
5554                                 .mask = NULL,
5555                         },
5556                         {
5557                                 .type = RTE_FLOW_ITEM_TYPE_END,
5558                         },
5559                 };
5560                 struct rte_flow_action q_actions[] = {
5561                         {
5562                                 .type = qrss->type,
5563                                 .conf = qrss->conf,
5564                         },
5565                         {
5566                                 .type = RTE_FLOW_ACTION_TYPE_END,
5567                         },
5568                 };
5569                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5570
5571                 /*
5572                  * Configure the tag item only if there is no meter subflow.
5573                  * Since tag is already marked in the meter suffix subflow
5574                  * we can just use the meter suffix items as is.
5575                  */
5576                 if (qrss_id) {
5577                         /* Not meter subflow. */
5578                         MLX5_ASSERT(!mtr_sfx);
5579                         /*
5580                          * Put unique id in prefix flow due to it is destroyed
5581                          * after suffix flow and id will be freed after there
5582                          * is no actual flows with this id and identifier
5583                          * reallocation becomes possible (for example, for
5584                          * other flows in other threads).
5585                          */
5586                         dev_flow->handle->split_flow_id = qrss_id;
5587                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5588                                                    error);
5589                         if (ret < 0)
5590                                 goto exit;
5591                         q_tag_spec.id = ret;
5592                 }
5593                 dev_flow = NULL;
5594                 /* Add suffix subflow to execute Q/RSS. */
5595                 flow_split_info->prefix_layers = layers;
5596                 flow_split_info->prefix_mark = 0;
5597                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5598                                               &q_attr, mtr_sfx ? items :
5599                                               q_items, q_actions,
5600                                               flow_split_info, error);
5601                 if (ret < 0)
5602                         goto exit;
5603                 /* qrss ID should be freed if failed. */
5604                 qrss_id = 0;
5605                 MLX5_ASSERT(dev_flow);
5606         }
5607
5608 exit:
5609         /*
5610          * We do not destroy the partially created sub_flows in case of error.
5611          * These ones are included into parent flow list and will be destroyed
5612          * by flow_drv_destroy.
5613          */
5614         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5615                         qrss_id);
5616         mlx5_free(ext_actions);
5617         return ret;
5618 }
5619
5620 /**
5621  * Create meter internal drop flow with the original pattern.
5622  *
5623  * @param dev
5624  *   Pointer to Ethernet device.
5625  * @param[in] flow
5626  *   Parent flow structure pointer.
5627  * @param[in] attr
5628  *   Flow rule attributes.
5629  * @param[in] items
5630  *   Pattern specification (list terminated by the END pattern item).
5631  * @param[in] flow_split_info
5632  *   Pointer to flow split info structure.
5633  * @param[in] fm
5634  *   Pointer to flow meter structure.
5635  * @param[out] error
5636  *   Perform verbose error reporting if not NULL.
5637  * @return
5638  *   0 on success, negative value otherwise
5639  */
5640 static uint32_t
5641 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5642                         struct rte_flow *flow,
5643                         const struct rte_flow_attr *attr,
5644                         const struct rte_flow_item items[],
5645                         struct mlx5_flow_split_info *flow_split_info,
5646                         struct mlx5_flow_meter_info *fm,
5647                         struct rte_flow_error *error)
5648 {
5649         struct mlx5_flow *dev_flow = NULL;
5650         struct rte_flow_attr drop_attr = *attr;
5651         struct rte_flow_action drop_actions[3];
5652         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5653
5654         MLX5_ASSERT(fm->drop_cnt);
5655         drop_actions[0].type =
5656                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5657         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5658         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5659         drop_actions[1].conf = NULL;
5660         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5661         drop_actions[2].conf = NULL;
5662         drop_split_info.external = false;
5663         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5664         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5665         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5666         return flow_create_split_inner(dev, flow, &dev_flow,
5667                                 &drop_attr, items, drop_actions,
5668                                 &drop_split_info, error);
5669 }
5670
5671 /**
5672  * The splitting for meter feature.
5673  *
5674  * - The meter flow will be split to two flows as prefix and
5675  *   suffix flow. The packets make sense only it pass the prefix
5676  *   meter action.
5677  *
5678  * - Reg_C_5 is used for the packet to match betweend prefix and
5679  *   suffix flow.
5680  *
5681  * @param dev
5682  *   Pointer to Ethernet device.
5683  * @param[in] flow
5684  *   Parent flow structure pointer.
5685  * @param[in] attr
5686  *   Flow rule attributes.
5687  * @param[in] items
5688  *   Pattern specification (list terminated by the END pattern item).
5689  * @param[in] actions
5690  *   Associated actions (list terminated by the END action).
5691  * @param[in] flow_split_info
5692  *   Pointer to flow split info structure.
5693  * @param[out] error
5694  *   Perform verbose error reporting if not NULL.
5695  * @return
5696  *   0 on success, negative value otherwise
5697  */
5698 static int
5699 flow_create_split_meter(struct rte_eth_dev *dev,
5700                         struct rte_flow *flow,
5701                         const struct rte_flow_attr *attr,
5702                         const struct rte_flow_item items[],
5703                         const struct rte_flow_action actions[],
5704                         struct mlx5_flow_split_info *flow_split_info,
5705                         struct rte_flow_error *error)
5706 {
5707         struct mlx5_priv *priv = dev->data->dev_private;
5708         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5709         struct rte_flow_action *sfx_actions = NULL;
5710         struct rte_flow_action *pre_actions = NULL;
5711         struct rte_flow_item *sfx_items = NULL;
5712         struct mlx5_flow *dev_flow = NULL;
5713         struct rte_flow_attr sfx_attr = *attr;
5714         struct mlx5_flow_meter_info *fm = NULL;
5715         uint8_t skip_scale_restore;
5716         bool has_mtr = false;
5717         bool has_modify = false;
5718         bool set_mtr_reg = true;
5719         uint32_t meter_id = 0;
5720         uint32_t mtr_idx = 0;
5721         uint32_t mtr_flow_id = 0;
5722         size_t act_size;
5723         size_t item_size;
5724         int actions_n = 0;
5725         int ret = 0;
5726
5727         if (priv->mtr_en)
5728                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5729                                                     &has_modify, &meter_id);
5730         if (has_mtr) {
5731                 if (flow->meter) {
5732                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5733                         if (!fm)
5734                                 return rte_flow_error_set(error, EINVAL,
5735                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5736                                                 NULL, "Meter not found.");
5737                 } else {
5738                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5739                         if (!fm)
5740                                 return rte_flow_error_set(error, EINVAL,
5741                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5742                                                 NULL, "Meter not found.");
5743                         ret = mlx5_flow_meter_attach(priv, fm,
5744                                                      &sfx_attr, error);
5745                         if (ret)
5746                                 return -rte_errno;
5747                         flow->meter = mtr_idx;
5748                 }
5749                 MLX5_ASSERT(wks);
5750                 wks->fm = fm;
5751                 /*
5752                  * If it isn't default-policy Meter, and
5753                  * 1. There's no action in flow to change
5754                  *    packet (modify/encap/decap etc.), OR
5755                  * 2. No drop count needed for this meter.
5756                  * no need to use regC to save meter id anymore.
5757                  */
5758                 if (!fm->def_policy && (!has_modify || !fm->drop_cnt))
5759                         set_mtr_reg = false;
5760                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5761                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5762                            sizeof(struct mlx5_rte_flow_action_set_tag);
5763                 /* Suffix items: tag, vlan, port id, end. */
5764 #define METER_SUFFIX_ITEM 4
5765                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5766                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5767                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5768                                           0, SOCKET_ID_ANY);
5769                 if (!sfx_actions)
5770                         return rte_flow_error_set(error, ENOMEM,
5771                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5772                                                   NULL, "no memory to split "
5773                                                   "meter flow");
5774                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5775                              act_size);
5776                 /* There's no suffix flow for meter of non-default policy. */
5777                 if (!fm->def_policy)
5778                         pre_actions = sfx_actions + 1;
5779                 else
5780                         pre_actions = sfx_actions + actions_n;
5781                 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr,
5782                                             items, sfx_items, actions,
5783                                             sfx_actions, pre_actions,
5784                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5785                                             error);
5786                 if (ret) {
5787                         ret = -rte_errno;
5788                         goto exit;
5789                 }
5790                 /* Add the prefix subflow. */
5791                 flow_split_info->prefix_mark = 0;
5792                 skip_scale_restore = flow_split_info->skip_scale;
5793                 flow_split_info->skip_scale |=
5794                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5795                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5796                                               attr, items, pre_actions,
5797                                               flow_split_info, error);
5798                 flow_split_info->skip_scale = skip_scale_restore;
5799                 if (ret) {
5800                         if (mtr_flow_id)
5801                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5802                         ret = -rte_errno;
5803                         goto exit;
5804                 }
5805                 if (mtr_flow_id) {
5806                         dev_flow->handle->split_flow_id = mtr_flow_id;
5807                         dev_flow->handle->is_meter_flow_id = 1;
5808                 }
5809                 if (!fm->def_policy) {
5810                         if (!set_mtr_reg && fm->drop_cnt)
5811                                 ret =
5812                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5813                                                         &sfx_attr, items,
5814                                                         flow_split_info,
5815                                                         fm, error);
5816                         goto exit;
5817                 }
5818                 /* Setting the sfx group atrr. */
5819                 sfx_attr.group = sfx_attr.transfer ?
5820                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5821                                  MLX5_FLOW_TABLE_LEVEL_METER;
5822                 flow_split_info->prefix_layers =
5823                                 flow_get_prefix_layer_flags(dev_flow);
5824                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5825                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5826         }
5827         /* Add the prefix subflow. */
5828         ret = flow_create_split_metadata(dev, flow,
5829                                          &sfx_attr, sfx_items ?
5830                                          sfx_items : items,
5831                                          sfx_actions ? sfx_actions : actions,
5832                                          flow_split_info, error);
5833 exit:
5834         if (sfx_actions)
5835                 mlx5_free(sfx_actions);
5836         return ret;
5837 }
5838
5839 /**
5840  * The splitting for sample feature.
5841  *
5842  * Once Sample action is detected in the action list, the flow actions should
5843  * be split into prefix sub flow and suffix sub flow.
5844  *
5845  * The original items remain in the prefix sub flow, all actions preceding the
5846  * sample action and the sample action itself will be copied to the prefix
5847  * sub flow, the actions following the sample action will be copied to the
5848  * suffix sub flow, Queue action always be located in the suffix sub flow.
5849  *
5850  * In order to make the packet from prefix sub flow matches with suffix sub
5851  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5852  * flow uses tag item with the unique flow id.
5853  *
5854  * @param dev
5855  *   Pointer to Ethernet device.
5856  * @param[in] flow
5857  *   Parent flow structure pointer.
5858  * @param[in] attr
5859  *   Flow rule attributes.
5860  * @param[in] items
5861  *   Pattern specification (list terminated by the END pattern item).
5862  * @param[in] actions
5863  *   Associated actions (list terminated by the END action).
5864  * @param[in] flow_split_info
5865  *   Pointer to flow split info structure.
5866  * @param[out] error
5867  *   Perform verbose error reporting if not NULL.
5868  * @return
5869  *   0 on success, negative value otherwise
5870  */
5871 static int
5872 flow_create_split_sample(struct rte_eth_dev *dev,
5873                          struct rte_flow *flow,
5874                          const struct rte_flow_attr *attr,
5875                          const struct rte_flow_item items[],
5876                          const struct rte_flow_action actions[],
5877                          struct mlx5_flow_split_info *flow_split_info,
5878                          struct rte_flow_error *error)
5879 {
5880         struct mlx5_priv *priv = dev->data->dev_private;
5881         struct rte_flow_action *sfx_actions = NULL;
5882         struct rte_flow_action *pre_actions = NULL;
5883         struct rte_flow_item *sfx_items = NULL;
5884         struct mlx5_flow *dev_flow = NULL;
5885         struct rte_flow_attr sfx_attr = *attr;
5886 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5887         struct mlx5_flow_dv_sample_resource *sample_res;
5888         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5889         struct mlx5_flow_tbl_resource *sfx_tbl;
5890 #endif
5891         size_t act_size;
5892         size_t item_size;
5893         uint32_t fdb_tx = 0;
5894         int32_t tag_id = 0;
5895         int actions_n = 0;
5896         int sample_action_pos;
5897         int qrss_action_pos;
5898         int add_tag = 0;
5899         int modify_after_mirror = 0;
5900         uint16_t jump_table = 0;
5901         const uint32_t next_ft_step = 1;
5902         int ret = 0;
5903
5904         if (priv->sampler_en)
5905                 actions_n = flow_check_match_action(actions, attr,
5906                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5907                                         &sample_action_pos, &qrss_action_pos,
5908                                         &modify_after_mirror);
5909         if (actions_n) {
5910                 /* The prefix actions must includes sample, tag, end. */
5911                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5912                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5913                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5914                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5915                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5916                                           item_size), 0, SOCKET_ID_ANY);
5917                 if (!sfx_actions)
5918                         return rte_flow_error_set(error, ENOMEM,
5919                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5920                                                   NULL, "no memory to split "
5921                                                   "sample flow");
5922                 /* The representor_id is UINT16_MAX for uplink. */
5923                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
5924                 /*
5925                  * When reg_c_preserve is set, metadata registers Cx preserve
5926                  * their value even through packet duplication.
5927                  */
5928                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
5929                 if (add_tag)
5930                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5931                                         + act_size);
5932                 if (modify_after_mirror)
5933                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
5934                                      next_ft_step;
5935                 pre_actions = sfx_actions + actions_n;
5936                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
5937                                                 actions, sfx_actions,
5938                                                 pre_actions, actions_n,
5939                                                 sample_action_pos,
5940                                                 qrss_action_pos, jump_table,
5941                                                 error);
5942                 if (tag_id < 0 || (add_tag && !tag_id)) {
5943                         ret = -rte_errno;
5944                         goto exit;
5945                 }
5946                 if (modify_after_mirror)
5947                         flow_split_info->skip_scale =
5948                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5949                 /* Add the prefix subflow. */
5950                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5951                                               items, pre_actions,
5952                                               flow_split_info, error);
5953                 if (ret) {
5954                         ret = -rte_errno;
5955                         goto exit;
5956                 }
5957                 dev_flow->handle->split_flow_id = tag_id;
5958 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5959                 if (!modify_after_mirror) {
5960                         /* Set the sfx group attr. */
5961                         sample_res = (struct mlx5_flow_dv_sample_resource *)
5962                                                 dev_flow->dv.sample_res;
5963                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
5964                                                 sample_res->normal_path_tbl;
5965                         sfx_tbl_data = container_of(sfx_tbl,
5966                                                 struct mlx5_flow_tbl_data_entry,
5967                                                 tbl);
5968                         sfx_attr.group = sfx_attr.transfer ?
5969                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
5970                 } else {
5971                         MLX5_ASSERT(attr->transfer);
5972                         sfx_attr.group = jump_table;
5973                 }
5974                 flow_split_info->prefix_layers =
5975                                 flow_get_prefix_layer_flags(dev_flow);
5976                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5977                 /* Suffix group level already be scaled with factor, set
5978                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
5979                  * again in translation.
5980                  */
5981                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5982 #endif
5983         }
5984         /* Add the suffix subflow. */
5985         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5986                                       sfx_items ? sfx_items : items,
5987                                       sfx_actions ? sfx_actions : actions,
5988                                       flow_split_info, error);
5989 exit:
5990         if (sfx_actions)
5991                 mlx5_free(sfx_actions);
5992         return ret;
5993 }
5994
5995 /**
5996  * Split the flow to subflow set. The splitters might be linked
5997  * in the chain, like this:
5998  * flow_create_split_outer() calls:
5999  *   flow_create_split_meter() calls:
6000  *     flow_create_split_metadata(meter_subflow_0) calls:
6001  *       flow_create_split_inner(metadata_subflow_0)
6002  *       flow_create_split_inner(metadata_subflow_1)
6003  *       flow_create_split_inner(metadata_subflow_2)
6004  *     flow_create_split_metadata(meter_subflow_1) calls:
6005  *       flow_create_split_inner(metadata_subflow_0)
6006  *       flow_create_split_inner(metadata_subflow_1)
6007  *       flow_create_split_inner(metadata_subflow_2)
6008  *
6009  * This provide flexible way to add new levels of flow splitting.
6010  * The all of successfully created subflows are included to the
6011  * parent flow dev_flow list.
6012  *
6013  * @param dev
6014  *   Pointer to Ethernet device.
6015  * @param[in] flow
6016  *   Parent flow structure pointer.
6017  * @param[in] attr
6018  *   Flow rule attributes.
6019  * @param[in] items
6020  *   Pattern specification (list terminated by the END pattern item).
6021  * @param[in] actions
6022  *   Associated actions (list terminated by the END action).
6023  * @param[in] flow_split_info
6024  *   Pointer to flow split info structure.
6025  * @param[out] error
6026  *   Perform verbose error reporting if not NULL.
6027  * @return
6028  *   0 on success, negative value otherwise
6029  */
6030 static int
6031 flow_create_split_outer(struct rte_eth_dev *dev,
6032                         struct rte_flow *flow,
6033                         const struct rte_flow_attr *attr,
6034                         const struct rte_flow_item items[],
6035                         const struct rte_flow_action actions[],
6036                         struct mlx5_flow_split_info *flow_split_info,
6037                         struct rte_flow_error *error)
6038 {
6039         int ret;
6040
6041         ret = flow_create_split_sample(dev, flow, attr, items,
6042                                        actions, flow_split_info, error);
6043         MLX5_ASSERT(ret <= 0);
6044         return ret;
6045 }
6046
6047 static inline struct mlx5_flow_tunnel *
6048 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6049 {
6050         struct mlx5_flow_tunnel *tunnel;
6051
6052 #pragma GCC diagnostic push
6053 #pragma GCC diagnostic ignored "-Wcast-qual"
6054         tunnel = (typeof(tunnel))flow->tunnel;
6055 #pragma GCC diagnostic pop
6056
6057         return tunnel;
6058 }
6059
6060 /**
6061  * Adjust flow RSS workspace if needed.
6062  *
6063  * @param wks
6064  *   Pointer to thread flow work space.
6065  * @param rss_desc
6066  *   Pointer to RSS descriptor.
6067  * @param[in] nrssq_num
6068  *   New RSS queue number.
6069  *
6070  * @return
6071  *   0 on success, -1 otherwise and rte_errno is set.
6072  */
6073 static int
6074 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6075                           struct mlx5_flow_rss_desc *rss_desc,
6076                           uint32_t nrssq_num)
6077 {
6078         if (likely(nrssq_num <= wks->rssq_num))
6079                 return 0;
6080         rss_desc->queue = realloc(rss_desc->queue,
6081                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6082         if (!rss_desc->queue) {
6083                 rte_errno = ENOMEM;
6084                 return -1;
6085         }
6086         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6087         return 0;
6088 }
6089
6090 /**
6091  * Create a flow and add it to @p list.
6092  *
6093  * @param dev
6094  *   Pointer to Ethernet device.
6095  * @param list
6096  *   Pointer to a TAILQ flow list. If this parameter NULL,
6097  *   no list insertion occurred, flow is just created,
6098  *   this is caller's responsibility to track the
6099  *   created flow.
6100  * @param[in] attr
6101  *   Flow rule attributes.
6102  * @param[in] items
6103  *   Pattern specification (list terminated by the END pattern item).
6104  * @param[in] actions
6105  *   Associated actions (list terminated by the END action).
6106  * @param[in] external
6107  *   This flow rule is created by request external to PMD.
6108  * @param[out] error
6109  *   Perform verbose error reporting if not NULL.
6110  *
6111  * @return
6112  *   A flow index on success, 0 otherwise and rte_errno is set.
6113  */
6114 static uint32_t
6115 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
6116                  const struct rte_flow_attr *attr,
6117                  const struct rte_flow_item items[],
6118                  const struct rte_flow_action original_actions[],
6119                  bool external, struct rte_flow_error *error)
6120 {
6121         struct mlx5_priv *priv = dev->data->dev_private;
6122         struct rte_flow *flow = NULL;
6123         struct mlx5_flow *dev_flow;
6124         const struct rte_flow_action_rss *rss = NULL;
6125         struct mlx5_translated_action_handle
6126                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6127         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6128         union {
6129                 struct mlx5_flow_expand_rss buf;
6130                 uint8_t buffer[2048];
6131         } expand_buffer;
6132         union {
6133                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6134                 uint8_t buffer[2048];
6135         } actions_rx;
6136         union {
6137                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6138                 uint8_t buffer[2048];
6139         } actions_hairpin_tx;
6140         union {
6141                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6142                 uint8_t buffer[2048];
6143         } items_tx;
6144         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6145         struct mlx5_flow_rss_desc *rss_desc;
6146         const struct rte_flow_action *p_actions_rx;
6147         uint32_t i;
6148         uint32_t idx = 0;
6149         int hairpin_flow;
6150         struct rte_flow_attr attr_tx = { .priority = 0 };
6151         const struct rte_flow_action *actions;
6152         struct rte_flow_action *translated_actions = NULL;
6153         struct mlx5_flow_tunnel *tunnel;
6154         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6155         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6156         struct mlx5_flow_split_info flow_split_info = {
6157                 .external = !!external,
6158                 .skip_scale = 0,
6159                 .flow_idx = 0,
6160                 .prefix_mark = 0,
6161                 .prefix_layers = 0,
6162                 .table_id = 0
6163         };
6164         int ret;
6165
6166         MLX5_ASSERT(wks);
6167         rss_desc = &wks->rss_desc;
6168         ret = flow_action_handles_translate(dev, original_actions,
6169                                             indir_actions,
6170                                             &indir_actions_n,
6171                                             &translated_actions, error);
6172         if (ret < 0) {
6173                 MLX5_ASSERT(translated_actions == NULL);
6174                 return 0;
6175         }
6176         actions = translated_actions ? translated_actions : original_actions;
6177         p_actions_rx = actions;
6178         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6179         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6180                                 external, hairpin_flow, error);
6181         if (ret < 0)
6182                 goto error_before_hairpin_split;
6183         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
6184         if (!flow) {
6185                 rte_errno = ENOMEM;
6186                 goto error_before_hairpin_split;
6187         }
6188         if (hairpin_flow > 0) {
6189                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6190                         rte_errno = EINVAL;
6191                         goto error_before_hairpin_split;
6192                 }
6193                 flow_hairpin_split(dev, actions, actions_rx.actions,
6194                                    actions_hairpin_tx.actions, items_tx.items,
6195                                    idx);
6196                 p_actions_rx = actions_rx.actions;
6197         }
6198         flow_split_info.flow_idx = idx;
6199         flow->drv_type = flow_get_drv_type(dev, attr);
6200         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6201                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6202         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6203         /* RSS Action only works on NIC RX domain */
6204         if (attr->ingress && !attr->transfer)
6205                 rss = flow_get_rss_action(dev, p_actions_rx);
6206         if (rss) {
6207                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6208                         return 0;
6209                 /*
6210                  * The following information is required by
6211                  * mlx5_flow_hashfields_adjust() in advance.
6212                  */
6213                 rss_desc->level = rss->level;
6214                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6215                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6216         }
6217         flow->dev_handles = 0;
6218         if (rss && rss->types) {
6219                 unsigned int graph_root;
6220
6221                 graph_root = find_graph_root(items, rss->level);
6222                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6223                                            items, rss->types,
6224                                            mlx5_support_expansion, graph_root);
6225                 MLX5_ASSERT(ret > 0 &&
6226                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6227                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6228                         for (i = 0; i < buf->entries; ++i)
6229                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6230                 }
6231         } else {
6232                 buf->entries = 1;
6233                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6234         }
6235         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6236                                                       indir_actions_n);
6237         for (i = 0; i < buf->entries; ++i) {
6238                 /* Initialize flow split data. */
6239                 flow_split_info.prefix_layers = 0;
6240                 flow_split_info.prefix_mark = 0;
6241                 flow_split_info.skip_scale = 0;
6242                 /*
6243                  * The splitter may create multiple dev_flows,
6244                  * depending on configuration. In the simplest
6245                  * case it just creates unmodified original flow.
6246                  */
6247                 ret = flow_create_split_outer(dev, flow, attr,
6248                                               buf->entry[i].pattern,
6249                                               p_actions_rx, &flow_split_info,
6250                                               error);
6251                 if (ret < 0)
6252                         goto error;
6253                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6254                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6255                                                            p_actions_rx,
6256                                                            idx,
6257                                                            wks->flows[0].tunnel,
6258                                                            &default_miss_ctx,
6259                                                            error);
6260                         if (ret < 0) {
6261                                 mlx5_free(default_miss_ctx.queue);
6262                                 goto error;
6263                         }
6264                 }
6265         }
6266         /* Create the tx flow. */
6267         if (hairpin_flow) {
6268                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6269                 attr_tx.ingress = 0;
6270                 attr_tx.egress = 1;
6271                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6272                                          actions_hairpin_tx.actions,
6273                                          idx, error);
6274                 if (!dev_flow)
6275                         goto error;
6276                 dev_flow->flow = flow;
6277                 dev_flow->external = 0;
6278                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6279                               dev_flow->handle, next);
6280                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6281                                          items_tx.items,
6282                                          actions_hairpin_tx.actions, error);
6283                 if (ret < 0)
6284                         goto error;
6285         }
6286         /*
6287          * Update the metadata register copy table. If extensive
6288          * metadata feature is enabled and registers are supported
6289          * we might create the extra rte_flow for each unique
6290          * MARK/FLAG action ID.
6291          *
6292          * The table is updated for ingress Flows only, because
6293          * the egress Flows belong to the different device and
6294          * copy table should be updated in peer NIC Rx domain.
6295          */
6296         if (attr->ingress &&
6297             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6298                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6299                 if (ret)
6300                         goto error;
6301         }
6302         /*
6303          * If the flow is external (from application) OR device is started,
6304          * OR mreg discover, then apply immediately.
6305          */
6306         if (external || dev->data->dev_started ||
6307             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6308              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6309                 ret = flow_drv_apply(dev, flow, error);
6310                 if (ret < 0)
6311                         goto error;
6312         }
6313         if (list) {
6314                 rte_spinlock_lock(&priv->flow_list_lock);
6315                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
6316                              flow, next);
6317                 rte_spinlock_unlock(&priv->flow_list_lock);
6318         }
6319         flow_rxq_flags_set(dev, flow);
6320         rte_free(translated_actions);
6321         tunnel = flow_tunnel_from_rule(wks->flows);
6322         if (tunnel) {
6323                 flow->tunnel = 1;
6324                 flow->tunnel_id = tunnel->tunnel_id;
6325                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6326                 mlx5_free(default_miss_ctx.queue);
6327         }
6328         mlx5_flow_pop_thread_workspace();
6329         return idx;
6330 error:
6331         MLX5_ASSERT(flow);
6332         ret = rte_errno; /* Save rte_errno before cleanup. */
6333         flow_mreg_del_copy_action(dev, flow);
6334         flow_drv_destroy(dev, flow);
6335         if (rss_desc->shared_rss)
6336                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6337                         mlx5_ipool_get
6338                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6339                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6340         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
6341         rte_errno = ret; /* Restore rte_errno. */
6342         ret = rte_errno;
6343         rte_errno = ret;
6344         mlx5_flow_pop_thread_workspace();
6345 error_before_hairpin_split:
6346         rte_free(translated_actions);
6347         return 0;
6348 }
6349
6350 /**
6351  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6352  * incoming packets to table 1.
6353  *
6354  * Other flow rules, requested for group n, will be created in
6355  * e-switch table n+1.
6356  * Jump action to e-switch group n will be created to group n+1.
6357  *
6358  * Used when working in switchdev mode, to utilise advantages of table 1
6359  * and above.
6360  *
6361  * @param dev
6362  *   Pointer to Ethernet device.
6363  *
6364  * @return
6365  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6366  */
6367 struct rte_flow *
6368 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6369 {
6370         const struct rte_flow_attr attr = {
6371                 .group = 0,
6372                 .priority = 0,
6373                 .ingress = 1,
6374                 .egress = 0,
6375                 .transfer = 1,
6376         };
6377         const struct rte_flow_item pattern = {
6378                 .type = RTE_FLOW_ITEM_TYPE_END,
6379         };
6380         struct rte_flow_action_jump jump = {
6381                 .group = 1,
6382         };
6383         const struct rte_flow_action actions[] = {
6384                 {
6385                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6386                         .conf = &jump,
6387                 },
6388                 {
6389                         .type = RTE_FLOW_ACTION_TYPE_END,
6390                 },
6391         };
6392         struct mlx5_priv *priv = dev->data->dev_private;
6393         struct rte_flow_error error;
6394
6395         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
6396                                                    &attr, &pattern,
6397                                                    actions, false, &error);
6398 }
6399
6400 /**
6401  * Validate a flow supported by the NIC.
6402  *
6403  * @see rte_flow_validate()
6404  * @see rte_flow_ops
6405  */
6406 int
6407 mlx5_flow_validate(struct rte_eth_dev *dev,
6408                    const struct rte_flow_attr *attr,
6409                    const struct rte_flow_item items[],
6410                    const struct rte_flow_action original_actions[],
6411                    struct rte_flow_error *error)
6412 {
6413         int hairpin_flow;
6414         struct mlx5_translated_action_handle
6415                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6416         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6417         const struct rte_flow_action *actions;
6418         struct rte_flow_action *translated_actions = NULL;
6419         int ret = flow_action_handles_translate(dev, original_actions,
6420                                                 indir_actions,
6421                                                 &indir_actions_n,
6422                                                 &translated_actions, error);
6423
6424         if (ret)
6425                 return ret;
6426         actions = translated_actions ? translated_actions : original_actions;
6427         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6428         ret = flow_drv_validate(dev, attr, items, actions,
6429                                 true, hairpin_flow, error);
6430         rte_free(translated_actions);
6431         return ret;
6432 }
6433
6434 /**
6435  * Create a flow.
6436  *
6437  * @see rte_flow_create()
6438  * @see rte_flow_ops
6439  */
6440 struct rte_flow *
6441 mlx5_flow_create(struct rte_eth_dev *dev,
6442                  const struct rte_flow_attr *attr,
6443                  const struct rte_flow_item items[],
6444                  const struct rte_flow_action actions[],
6445                  struct rte_flow_error *error)
6446 {
6447         struct mlx5_priv *priv = dev->data->dev_private;
6448
6449         /*
6450          * If the device is not started yet, it is not allowed to created a
6451          * flow from application. PMD default flows and traffic control flows
6452          * are not affected.
6453          */
6454         if (unlikely(!dev->data->dev_started)) {
6455                 DRV_LOG(DEBUG, "port %u is not started when "
6456                         "inserting a flow", dev->data->port_id);
6457                 rte_flow_error_set(error, ENODEV,
6458                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6459                                    NULL,
6460                                    "port not started");
6461                 return NULL;
6462         }
6463
6464         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
6465                                   attr, items, actions, true, error);
6466 }
6467
6468 /**
6469  * Destroy a flow in a list.
6470  *
6471  * @param dev
6472  *   Pointer to Ethernet device.
6473  * @param list
6474  *   Pointer to the Indexed flow list. If this parameter NULL,
6475  *   there is no flow removal from the list. Be noted that as
6476  *   flow is add to the indexed list, memory of the indexed
6477  *   list points to maybe changed as flow destroyed.
6478  * @param[in] flow_idx
6479  *   Index of flow to destroy.
6480  */
6481 static void
6482 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
6483                   uint32_t flow_idx)
6484 {
6485         struct mlx5_priv *priv = dev->data->dev_private;
6486         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6487                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
6488
6489         if (!flow)
6490                 return;
6491         /*
6492          * Update RX queue flags only if port is started, otherwise it is
6493          * already clean.
6494          */
6495         if (dev->data->dev_started)
6496                 flow_rxq_flags_trim(dev, flow);
6497         flow_drv_destroy(dev, flow);
6498         if (list) {
6499                 rte_spinlock_lock(&priv->flow_list_lock);
6500                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
6501                              flow_idx, flow, next);
6502                 rte_spinlock_unlock(&priv->flow_list_lock);
6503         }
6504         if (flow->tunnel) {
6505                 struct mlx5_flow_tunnel *tunnel;
6506
6507                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6508                 RTE_VERIFY(tunnel);
6509                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6510                         mlx5_flow_tunnel_free(dev, tunnel);
6511         }
6512         flow_mreg_del_copy_action(dev, flow);
6513         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6514 }
6515
6516 /**
6517  * Destroy all flows.
6518  *
6519  * @param dev
6520  *   Pointer to Ethernet device.
6521  * @param list
6522  *   Pointer to the Indexed flow list.
6523  * @param active
6524  *   If flushing is called avtively.
6525  */
6526 void
6527 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
6528 {
6529         uint32_t num_flushed = 0;
6530
6531         while (*list) {
6532                 flow_list_destroy(dev, list, *list);
6533                 num_flushed++;
6534         }
6535         if (active) {
6536                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6537                         dev->data->port_id, num_flushed);
6538         }
6539 }
6540
6541 /**
6542  * Stop all default actions for flows.
6543  *
6544  * @param dev
6545  *   Pointer to Ethernet device.
6546  */
6547 void
6548 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6549 {
6550         flow_mreg_del_default_copy_action(dev);
6551         flow_rxq_flags_clear(dev);
6552 }
6553
6554 /**
6555  * Start all default actions for flows.
6556  *
6557  * @param dev
6558  *   Pointer to Ethernet device.
6559  * @return
6560  *   0 on success, a negative errno value otherwise and rte_errno is set.
6561  */
6562 int
6563 mlx5_flow_start_default(struct rte_eth_dev *dev)
6564 {
6565         struct rte_flow_error error;
6566
6567         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6568         return flow_mreg_add_default_copy_action(dev, &error);
6569 }
6570
6571 /**
6572  * Release key of thread specific flow workspace data.
6573  */
6574 void
6575 flow_release_workspace(void *data)
6576 {
6577         struct mlx5_flow_workspace *wks = data;
6578         struct mlx5_flow_workspace *next;
6579
6580         while (wks) {
6581                 next = wks->next;
6582                 free(wks->rss_desc.queue);
6583                 free(wks);
6584                 wks = next;
6585         }
6586 }
6587
6588 /**
6589  * Get thread specific current flow workspace.
6590  *
6591  * @return pointer to thread specific flow workspace data, NULL on error.
6592  */
6593 struct mlx5_flow_workspace*
6594 mlx5_flow_get_thread_workspace(void)
6595 {
6596         struct mlx5_flow_workspace *data;
6597
6598         data = mlx5_flow_os_get_specific_workspace();
6599         MLX5_ASSERT(data && data->inuse);
6600         if (!data || !data->inuse)
6601                 DRV_LOG(ERR, "flow workspace not initialized.");
6602         return data;
6603 }
6604
6605 /**
6606  * Allocate and init new flow workspace.
6607  *
6608  * @return pointer to flow workspace data, NULL on error.
6609  */
6610 static struct mlx5_flow_workspace*
6611 flow_alloc_thread_workspace(void)
6612 {
6613         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6614
6615         if (!data) {
6616                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6617                         "memory.");
6618                 return NULL;
6619         }
6620         data->rss_desc.queue = calloc(1,
6621                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6622         if (!data->rss_desc.queue)
6623                 goto err;
6624         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6625         return data;
6626 err:
6627         if (data->rss_desc.queue)
6628                 free(data->rss_desc.queue);
6629         free(data);
6630         return NULL;
6631 }
6632
6633 /**
6634  * Get new thread specific flow workspace.
6635  *
6636  * If current workspace inuse, create new one and set as current.
6637  *
6638  * @return pointer to thread specific flow workspace data, NULL on error.
6639  */
6640 static struct mlx5_flow_workspace*
6641 mlx5_flow_push_thread_workspace(void)
6642 {
6643         struct mlx5_flow_workspace *curr;
6644         struct mlx5_flow_workspace *data;
6645
6646         curr = mlx5_flow_os_get_specific_workspace();
6647         if (!curr) {
6648                 data = flow_alloc_thread_workspace();
6649                 if (!data)
6650                         return NULL;
6651         } else if (!curr->inuse) {
6652                 data = curr;
6653         } else if (curr->next) {
6654                 data = curr->next;
6655         } else {
6656                 data = flow_alloc_thread_workspace();
6657                 if (!data)
6658                         return NULL;
6659                 curr->next = data;
6660                 data->prev = curr;
6661         }
6662         data->inuse = 1;
6663         data->flow_idx = 0;
6664         /* Set as current workspace */
6665         if (mlx5_flow_os_set_specific_workspace(data))
6666                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6667         return data;
6668 }
6669
6670 /**
6671  * Close current thread specific flow workspace.
6672  *
6673  * If previous workspace available, set it as current.
6674  *
6675  * @return pointer to thread specific flow workspace data, NULL on error.
6676  */
6677 static void
6678 mlx5_flow_pop_thread_workspace(void)
6679 {
6680         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6681
6682         if (!data)
6683                 return;
6684         if (!data->inuse) {
6685                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6686                 return;
6687         }
6688         data->inuse = 0;
6689         if (!data->prev)
6690                 return;
6691         if (mlx5_flow_os_set_specific_workspace(data->prev))
6692                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6693 }
6694
6695 /**
6696  * Verify the flow list is empty
6697  *
6698  * @param dev
6699  *  Pointer to Ethernet device.
6700  *
6701  * @return the number of flows not released.
6702  */
6703 int
6704 mlx5_flow_verify(struct rte_eth_dev *dev)
6705 {
6706         struct mlx5_priv *priv = dev->data->dev_private;
6707         struct rte_flow *flow;
6708         uint32_t idx;
6709         int ret = 0;
6710
6711         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6712                       flow, next) {
6713                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6714                         dev->data->port_id, (void *)flow);
6715                 ++ret;
6716         }
6717         return ret;
6718 }
6719
6720 /**
6721  * Enable default hairpin egress flow.
6722  *
6723  * @param dev
6724  *   Pointer to Ethernet device.
6725  * @param queue
6726  *   The queue index.
6727  *
6728  * @return
6729  *   0 on success, a negative errno value otherwise and rte_errno is set.
6730  */
6731 int
6732 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6733                             uint32_t queue)
6734 {
6735         struct mlx5_priv *priv = dev->data->dev_private;
6736         const struct rte_flow_attr attr = {
6737                 .egress = 1,
6738                 .priority = 0,
6739         };
6740         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6741                 .queue = queue,
6742         };
6743         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6744                 .queue = UINT32_MAX,
6745         };
6746         struct rte_flow_item items[] = {
6747                 {
6748                         .type = (enum rte_flow_item_type)
6749                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6750                         .spec = &queue_spec,
6751                         .last = NULL,
6752                         .mask = &queue_mask,
6753                 },
6754                 {
6755                         .type = RTE_FLOW_ITEM_TYPE_END,
6756                 },
6757         };
6758         struct rte_flow_action_jump jump = {
6759                 .group = MLX5_HAIRPIN_TX_TABLE,
6760         };
6761         struct rte_flow_action actions[2];
6762         uint32_t flow_idx;
6763         struct rte_flow_error error;
6764
6765         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6766         actions[0].conf = &jump;
6767         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6768         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6769                                 &attr, items, actions, false, &error);
6770         if (!flow_idx) {
6771                 DRV_LOG(DEBUG,
6772                         "Failed to create ctrl flow: rte_errno(%d),"
6773                         " type(%d), message(%s)",
6774                         rte_errno, error.type,
6775                         error.message ? error.message : " (no stated reason)");
6776                 return -rte_errno;
6777         }
6778         return 0;
6779 }
6780
6781 /**
6782  * Enable a control flow configured from the control plane.
6783  *
6784  * @param dev
6785  *   Pointer to Ethernet device.
6786  * @param eth_spec
6787  *   An Ethernet flow spec to apply.
6788  * @param eth_mask
6789  *   An Ethernet flow mask to apply.
6790  * @param vlan_spec
6791  *   A VLAN flow spec to apply.
6792  * @param vlan_mask
6793  *   A VLAN flow mask to apply.
6794  *
6795  * @return
6796  *   0 on success, a negative errno value otherwise and rte_errno is set.
6797  */
6798 int
6799 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6800                     struct rte_flow_item_eth *eth_spec,
6801                     struct rte_flow_item_eth *eth_mask,
6802                     struct rte_flow_item_vlan *vlan_spec,
6803                     struct rte_flow_item_vlan *vlan_mask)
6804 {
6805         struct mlx5_priv *priv = dev->data->dev_private;
6806         const struct rte_flow_attr attr = {
6807                 .ingress = 1,
6808                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6809         };
6810         struct rte_flow_item items[] = {
6811                 {
6812                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6813                         .spec = eth_spec,
6814                         .last = NULL,
6815                         .mask = eth_mask,
6816                 },
6817                 {
6818                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6819                                               RTE_FLOW_ITEM_TYPE_END,
6820                         .spec = vlan_spec,
6821                         .last = NULL,
6822                         .mask = vlan_mask,
6823                 },
6824                 {
6825                         .type = RTE_FLOW_ITEM_TYPE_END,
6826                 },
6827         };
6828         uint16_t queue[priv->reta_idx_n];
6829         struct rte_flow_action_rss action_rss = {
6830                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6831                 .level = 0,
6832                 .types = priv->rss_conf.rss_hf,
6833                 .key_len = priv->rss_conf.rss_key_len,
6834                 .queue_num = priv->reta_idx_n,
6835                 .key = priv->rss_conf.rss_key,
6836                 .queue = queue,
6837         };
6838         struct rte_flow_action actions[] = {
6839                 {
6840                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6841                         .conf = &action_rss,
6842                 },
6843                 {
6844                         .type = RTE_FLOW_ACTION_TYPE_END,
6845                 },
6846         };
6847         uint32_t flow_idx;
6848         struct rte_flow_error error;
6849         unsigned int i;
6850
6851         if (!priv->reta_idx_n || !priv->rxqs_n) {
6852                 return 0;
6853         }
6854         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6855                 action_rss.types = 0;
6856         for (i = 0; i != priv->reta_idx_n; ++i)
6857                 queue[i] = (*priv->reta_idx)[i];
6858         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6859                                 &attr, items, actions, false, &error);
6860         if (!flow_idx)
6861                 return -rte_errno;
6862         return 0;
6863 }
6864
6865 /**
6866  * Enable a flow control configured from the control plane.
6867  *
6868  * @param dev
6869  *   Pointer to Ethernet device.
6870  * @param eth_spec
6871  *   An Ethernet flow spec to apply.
6872  * @param eth_mask
6873  *   An Ethernet flow mask to apply.
6874  *
6875  * @return
6876  *   0 on success, a negative errno value otherwise and rte_errno is set.
6877  */
6878 int
6879 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6880                struct rte_flow_item_eth *eth_spec,
6881                struct rte_flow_item_eth *eth_mask)
6882 {
6883         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6884 }
6885
6886 /**
6887  * Create default miss flow rule matching lacp traffic
6888  *
6889  * @param dev
6890  *   Pointer to Ethernet device.
6891  * @param eth_spec
6892  *   An Ethernet flow spec to apply.
6893  *
6894  * @return
6895  *   0 on success, a negative errno value otherwise and rte_errno is set.
6896  */
6897 int
6898 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6899 {
6900         struct mlx5_priv *priv = dev->data->dev_private;
6901         /*
6902          * The LACP matching is done by only using ether type since using
6903          * a multicast dst mac causes kernel to give low priority to this flow.
6904          */
6905         static const struct rte_flow_item_eth lacp_spec = {
6906                 .type = RTE_BE16(0x8809),
6907         };
6908         static const struct rte_flow_item_eth lacp_mask = {
6909                 .type = 0xffff,
6910         };
6911         const struct rte_flow_attr attr = {
6912                 .ingress = 1,
6913         };
6914         struct rte_flow_item items[] = {
6915                 {
6916                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6917                         .spec = &lacp_spec,
6918                         .mask = &lacp_mask,
6919                 },
6920                 {
6921                         .type = RTE_FLOW_ITEM_TYPE_END,
6922                 },
6923         };
6924         struct rte_flow_action actions[] = {
6925                 {
6926                         .type = (enum rte_flow_action_type)
6927                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6928                 },
6929                 {
6930                         .type = RTE_FLOW_ACTION_TYPE_END,
6931                 },
6932         };
6933         struct rte_flow_error error;
6934         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6935                                 &attr, items, actions, false, &error);
6936
6937         if (!flow_idx)
6938                 return -rte_errno;
6939         return 0;
6940 }
6941
6942 /**
6943  * Destroy a flow.
6944  *
6945  * @see rte_flow_destroy()
6946  * @see rte_flow_ops
6947  */
6948 int
6949 mlx5_flow_destroy(struct rte_eth_dev *dev,
6950                   struct rte_flow *flow,
6951                   struct rte_flow_error *error __rte_unused)
6952 {
6953         struct mlx5_priv *priv = dev->data->dev_private;
6954
6955         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6956         return 0;
6957 }
6958
6959 /**
6960  * Destroy all flows.
6961  *
6962  * @see rte_flow_flush()
6963  * @see rte_flow_ops
6964  */
6965 int
6966 mlx5_flow_flush(struct rte_eth_dev *dev,
6967                 struct rte_flow_error *error __rte_unused)
6968 {
6969         struct mlx5_priv *priv = dev->data->dev_private;
6970
6971         mlx5_flow_list_flush(dev, &priv->flows, false);
6972         return 0;
6973 }
6974
6975 /**
6976  * Isolated mode.
6977  *
6978  * @see rte_flow_isolate()
6979  * @see rte_flow_ops
6980  */
6981 int
6982 mlx5_flow_isolate(struct rte_eth_dev *dev,
6983                   int enable,
6984                   struct rte_flow_error *error)
6985 {
6986         struct mlx5_priv *priv = dev->data->dev_private;
6987
6988         if (dev->data->dev_started) {
6989                 rte_flow_error_set(error, EBUSY,
6990                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6991                                    NULL,
6992                                    "port must be stopped first");
6993                 return -rte_errno;
6994         }
6995         priv->isolated = !!enable;
6996         if (enable)
6997                 dev->dev_ops = &mlx5_dev_ops_isolate;
6998         else
6999                 dev->dev_ops = &mlx5_dev_ops;
7000
7001         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7002         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7003
7004         return 0;
7005 }
7006
7007 /**
7008  * Query a flow.
7009  *
7010  * @see rte_flow_query()
7011  * @see rte_flow_ops
7012  */
7013 static int
7014 flow_drv_query(struct rte_eth_dev *dev,
7015                uint32_t flow_idx,
7016                const struct rte_flow_action *actions,
7017                void *data,
7018                struct rte_flow_error *error)
7019 {
7020         struct mlx5_priv *priv = dev->data->dev_private;
7021         const struct mlx5_flow_driver_ops *fops;
7022         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
7023                                                [MLX5_IPOOL_RTE_FLOW],
7024                                                flow_idx);
7025         enum mlx5_flow_drv_type ftype;
7026
7027         if (!flow) {
7028                 return rte_flow_error_set(error, ENOENT,
7029                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7030                           NULL,
7031                           "invalid flow handle");
7032         }
7033         ftype = flow->drv_type;
7034         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7035         fops = flow_get_drv_ops(ftype);
7036
7037         return fops->query(dev, flow, actions, data, error);
7038 }
7039
7040 /**
7041  * Query a flow.
7042  *
7043  * @see rte_flow_query()
7044  * @see rte_flow_ops
7045  */
7046 int
7047 mlx5_flow_query(struct rte_eth_dev *dev,
7048                 struct rte_flow *flow,
7049                 const struct rte_flow_action *actions,
7050                 void *data,
7051                 struct rte_flow_error *error)
7052 {
7053         int ret;
7054
7055         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7056                              error);
7057         if (ret < 0)
7058                 return ret;
7059         return 0;
7060 }
7061
7062 /**
7063  * Get rte_flow callbacks.
7064  *
7065  * @param dev
7066  *   Pointer to Ethernet device structure.
7067  * @param ops
7068  *   Pointer to operation-specific structure.
7069  *
7070  * @return 0
7071  */
7072 int
7073 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7074                   const struct rte_flow_ops **ops)
7075 {
7076         *ops = &mlx5_flow_ops;
7077         return 0;
7078 }
7079
7080 /**
7081  * Validate meter policy actions.
7082  * Dispatcher for action type specific validation.
7083  *
7084  * @param[in] dev
7085  *   Pointer to the Ethernet device structure.
7086  * @param[in] action
7087  *   The meter policy action object to validate.
7088  * @param[in] attr
7089  *   Attributes of flow to determine steering domain.
7090  * @param[out] is_rss
7091  *   Is RSS or not.
7092  * @param[out] domain_bitmap
7093  *   Domain bitmap.
7094  * @param[out] is_def_policy
7095  *   Is default policy or not.
7096  * @param[out] error
7097  *   Perform verbose error reporting if not NULL. Initialized in case of
7098  *   error only.
7099  *
7100  * @return
7101  *   0 on success, otherwise negative errno value.
7102  */
7103 int
7104 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7105                         const struct rte_flow_action *actions[RTE_COLORS],
7106                         struct rte_flow_attr *attr,
7107                         bool *is_rss,
7108                         uint8_t *domain_bitmap,
7109                         bool *is_def_policy,
7110                         struct rte_mtr_error *error)
7111 {
7112         const struct mlx5_flow_driver_ops *fops;
7113
7114         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7115         return fops->validate_mtr_acts(dev, actions, attr,
7116                         is_rss, domain_bitmap, is_def_policy, error);
7117 }
7118
7119 /**
7120  * Destroy the meter table set.
7121  *
7122  * @param[in] dev
7123  *   Pointer to Ethernet device.
7124  * @param[in] mtr_policy
7125  *   Meter policy struct.
7126  */
7127 void
7128 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7129                       struct mlx5_flow_meter_policy *mtr_policy)
7130 {
7131         const struct mlx5_flow_driver_ops *fops;
7132
7133         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7134         fops->destroy_mtr_acts(dev, mtr_policy);
7135 }
7136
7137 /**
7138  * Create policy action, lock free,
7139  * (mutex should be acquired by caller).
7140  * Dispatcher for action type specific call.
7141  *
7142  * @param[in] dev
7143  *   Pointer to the Ethernet device structure.
7144  * @param[in] mtr_policy
7145  *   Meter policy struct.
7146  * @param[in] action
7147  *   Action specification used to create meter actions.
7148  * @param[out] error
7149  *   Perform verbose error reporting if not NULL. Initialized in case of
7150  *   error only.
7151  *
7152  * @return
7153  *   0 on success, otherwise negative errno value.
7154  */
7155 int
7156 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7157                       struct mlx5_flow_meter_policy *mtr_policy,
7158                       const struct rte_flow_action *actions[RTE_COLORS],
7159                       struct rte_mtr_error *error)
7160 {
7161         const struct mlx5_flow_driver_ops *fops;
7162
7163         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7164         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7165 }
7166
7167 /**
7168  * Create policy rules, lock free,
7169  * (mutex should be acquired by caller).
7170  * Dispatcher for action type specific call.
7171  *
7172  * @param[in] dev
7173  *   Pointer to the Ethernet device structure.
7174  * @param[in] mtr_policy
7175  *   Meter policy struct.
7176  *
7177  * @return
7178  *   0 on success, -1 otherwise.
7179  */
7180 int
7181 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7182                              struct mlx5_flow_meter_policy *mtr_policy)
7183 {
7184         const struct mlx5_flow_driver_ops *fops;
7185
7186         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7187         return fops->create_policy_rules(dev, mtr_policy);
7188 }
7189
7190 /**
7191  * Destroy policy rules, lock free,
7192  * (mutex should be acquired by caller).
7193  * Dispatcher for action type specific call.
7194  *
7195  * @param[in] dev
7196  *   Pointer to the Ethernet device structure.
7197  * @param[in] mtr_policy
7198  *   Meter policy struct.
7199  */
7200 void
7201 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7202                              struct mlx5_flow_meter_policy *mtr_policy)
7203 {
7204         const struct mlx5_flow_driver_ops *fops;
7205
7206         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7207         fops->destroy_policy_rules(dev, mtr_policy);
7208 }
7209
7210 /**
7211  * Destroy the default policy table set.
7212  *
7213  * @param[in] dev
7214  *   Pointer to Ethernet device.
7215  */
7216 void
7217 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7218 {
7219         const struct mlx5_flow_driver_ops *fops;
7220
7221         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7222         fops->destroy_def_policy(dev);
7223 }
7224
7225 /**
7226  * Destroy the default policy table set.
7227  *
7228  * @param[in] dev
7229  *   Pointer to Ethernet device.
7230  *
7231  * @return
7232  *   0 on success, -1 otherwise.
7233  */
7234 int
7235 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7236 {
7237         const struct mlx5_flow_driver_ops *fops;
7238
7239         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7240         return fops->create_def_policy(dev);
7241 }
7242
7243 /**
7244  * Create the needed meter and suffix tables.
7245  *
7246  * @param[in] dev
7247  *   Pointer to Ethernet device.
7248  *
7249  * @return
7250  *   0 on success, -1 otherwise.
7251  */
7252 int
7253 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7254                         struct mlx5_flow_meter_info *fm,
7255                         uint32_t mtr_idx,
7256                         uint8_t domain_bitmap)
7257 {
7258         const struct mlx5_flow_driver_ops *fops;
7259
7260         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7261         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7262 }
7263
7264 /**
7265  * Destroy the meter table set.
7266  *
7267  * @param[in] dev
7268  *   Pointer to Ethernet device.
7269  * @param[in] tbl
7270  *   Pointer to the meter table set.
7271  */
7272 void
7273 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7274                            struct mlx5_flow_meter_info *fm)
7275 {
7276         const struct mlx5_flow_driver_ops *fops;
7277
7278         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7279         fops->destroy_mtr_tbls(dev, fm);
7280 }
7281
7282 /**
7283  * Destroy the global meter drop table.
7284  *
7285  * @param[in] dev
7286  *   Pointer to Ethernet device.
7287  */
7288 void
7289 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7290 {
7291         const struct mlx5_flow_driver_ops *fops;
7292
7293         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7294         fops->destroy_mtr_drop_tbls(dev);
7295 }
7296
7297 /**
7298  * Destroy the sub policy table with RX queue.
7299  *
7300  * @param[in] dev
7301  *   Pointer to Ethernet device.
7302  * @param[in] mtr_policy
7303  *   Pointer to meter policy table.
7304  */
7305 void
7306 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
7307                 struct mlx5_flow_meter_policy *mtr_policy)
7308 {
7309         const struct mlx5_flow_driver_ops *fops;
7310
7311         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7312         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
7313 }
7314
7315 /**
7316  * Allocate the needed aso flow meter id.
7317  *
7318  * @param[in] dev
7319  *   Pointer to Ethernet device.
7320  *
7321  * @return
7322  *   Index to aso flow meter on success, NULL otherwise.
7323  */
7324 uint32_t
7325 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7326 {
7327         const struct mlx5_flow_driver_ops *fops;
7328
7329         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7330         return fops->create_meter(dev);
7331 }
7332
7333 /**
7334  * Free the aso flow meter id.
7335  *
7336  * @param[in] dev
7337  *   Pointer to Ethernet device.
7338  * @param[in] mtr_idx
7339  *  Index to aso flow meter to be free.
7340  *
7341  * @return
7342  *   0 on success.
7343  */
7344 void
7345 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7346 {
7347         const struct mlx5_flow_driver_ops *fops;
7348
7349         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7350         fops->free_meter(dev, mtr_idx);
7351 }
7352
7353 /**
7354  * Allocate a counter.
7355  *
7356  * @param[in] dev
7357  *   Pointer to Ethernet device structure.
7358  *
7359  * @return
7360  *   Index to allocated counter  on success, 0 otherwise.
7361  */
7362 uint32_t
7363 mlx5_counter_alloc(struct rte_eth_dev *dev)
7364 {
7365         const struct mlx5_flow_driver_ops *fops;
7366         struct rte_flow_attr attr = { .transfer = 0 };
7367
7368         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7369                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7370                 return fops->counter_alloc(dev);
7371         }
7372         DRV_LOG(ERR,
7373                 "port %u counter allocate is not supported.",
7374                  dev->data->port_id);
7375         return 0;
7376 }
7377
7378 /**
7379  * Free a counter.
7380  *
7381  * @param[in] dev
7382  *   Pointer to Ethernet device structure.
7383  * @param[in] cnt
7384  *   Index to counter to be free.
7385  */
7386 void
7387 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7388 {
7389         const struct mlx5_flow_driver_ops *fops;
7390         struct rte_flow_attr attr = { .transfer = 0 };
7391
7392         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7393                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7394                 fops->counter_free(dev, cnt);
7395                 return;
7396         }
7397         DRV_LOG(ERR,
7398                 "port %u counter free is not supported.",
7399                  dev->data->port_id);
7400 }
7401
7402 /**
7403  * Query counter statistics.
7404  *
7405  * @param[in] dev
7406  *   Pointer to Ethernet device structure.
7407  * @param[in] cnt
7408  *   Index to counter to query.
7409  * @param[in] clear
7410  *   Set to clear counter statistics.
7411  * @param[out] pkts
7412  *   The counter hits packets number to save.
7413  * @param[out] bytes
7414  *   The counter hits bytes number to save.
7415  *
7416  * @return
7417  *   0 on success, a negative errno value otherwise.
7418  */
7419 int
7420 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7421                    bool clear, uint64_t *pkts, uint64_t *bytes)
7422 {
7423         const struct mlx5_flow_driver_ops *fops;
7424         struct rte_flow_attr attr = { .transfer = 0 };
7425
7426         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7427                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7428                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7429         }
7430         DRV_LOG(ERR,
7431                 "port %u counter query is not supported.",
7432                  dev->data->port_id);
7433         return -ENOTSUP;
7434 }
7435
7436 /**
7437  * Allocate a new memory for the counter values wrapped by all the needed
7438  * management.
7439  *
7440  * @param[in] sh
7441  *   Pointer to mlx5_dev_ctx_shared object.
7442  *
7443  * @return
7444  *   0 on success, a negative errno value otherwise.
7445  */
7446 static int
7447 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7448 {
7449         struct mlx5_devx_mkey_attr mkey_attr;
7450         struct mlx5_counter_stats_mem_mng *mem_mng;
7451         volatile struct flow_counter_stats *raw_data;
7452         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7453         int size = (sizeof(struct flow_counter_stats) *
7454                         MLX5_COUNTERS_PER_POOL +
7455                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7456                         sizeof(struct mlx5_counter_stats_mem_mng);
7457         size_t pgsize = rte_mem_page_size();
7458         uint8_t *mem;
7459         int i;
7460
7461         if (pgsize == (size_t)-1) {
7462                 DRV_LOG(ERR, "Failed to get mem page size");
7463                 rte_errno = ENOMEM;
7464                 return -ENOMEM;
7465         }
7466         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7467         if (!mem) {
7468                 rte_errno = ENOMEM;
7469                 return -ENOMEM;
7470         }
7471         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7472         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7473         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7474                                                  IBV_ACCESS_LOCAL_WRITE);
7475         if (!mem_mng->umem) {
7476                 rte_errno = errno;
7477                 mlx5_free(mem);
7478                 return -rte_errno;
7479         }
7480         memset(&mkey_attr, 0, sizeof(mkey_attr));
7481         mkey_attr.addr = (uintptr_t)mem;
7482         mkey_attr.size = size;
7483         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7484         mkey_attr.pd = sh->pdn;
7485         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7486         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7487         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7488         if (!mem_mng->dm) {
7489                 mlx5_os_umem_dereg(mem_mng->umem);
7490                 rte_errno = errno;
7491                 mlx5_free(mem);
7492                 return -rte_errno;
7493         }
7494         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7495         raw_data = (volatile struct flow_counter_stats *)mem;
7496         for (i = 0; i < raws_n; ++i) {
7497                 mem_mng->raws[i].mem_mng = mem_mng;
7498                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7499         }
7500         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7501                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7502                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7503                                  next);
7504         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7505         sh->cmng.mem_mng = mem_mng;
7506         return 0;
7507 }
7508
7509 /**
7510  * Set the statistic memory to the new counter pool.
7511  *
7512  * @param[in] sh
7513  *   Pointer to mlx5_dev_ctx_shared object.
7514  * @param[in] pool
7515  *   Pointer to the pool to set the statistic memory.
7516  *
7517  * @return
7518  *   0 on success, a negative errno value otherwise.
7519  */
7520 static int
7521 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7522                                struct mlx5_flow_counter_pool *pool)
7523 {
7524         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7525         /* Resize statistic memory once used out. */
7526         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7527             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7528                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7529                 return -1;
7530         }
7531         rte_spinlock_lock(&pool->sl);
7532         pool->raw = cmng->mem_mng->raws + pool->index %
7533                     MLX5_CNT_CONTAINER_RESIZE;
7534         rte_spinlock_unlock(&pool->sl);
7535         pool->raw_hw = NULL;
7536         return 0;
7537 }
7538
7539 #define MLX5_POOL_QUERY_FREQ_US 1000000
7540
7541 /**
7542  * Set the periodic procedure for triggering asynchronous batch queries for all
7543  * the counter pools.
7544  *
7545  * @param[in] sh
7546  *   Pointer to mlx5_dev_ctx_shared object.
7547  */
7548 void
7549 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7550 {
7551         uint32_t pools_n, us;
7552
7553         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7554         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7555         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7556         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7557                 sh->cmng.query_thread_on = 0;
7558                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7559         } else {
7560                 sh->cmng.query_thread_on = 1;
7561         }
7562 }
7563
7564 /**
7565  * The periodic procedure for triggering asynchronous batch queries for all the
7566  * counter pools. This function is probably called by the host thread.
7567  *
7568  * @param[in] arg
7569  *   The parameter for the alarm process.
7570  */
7571 void
7572 mlx5_flow_query_alarm(void *arg)
7573 {
7574         struct mlx5_dev_ctx_shared *sh = arg;
7575         int ret;
7576         uint16_t pool_index = sh->cmng.pool_index;
7577         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7578         struct mlx5_flow_counter_pool *pool;
7579         uint16_t n_valid;
7580
7581         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7582                 goto set_alarm;
7583         rte_spinlock_lock(&cmng->pool_update_sl);
7584         pool = cmng->pools[pool_index];
7585         n_valid = cmng->n_valid;
7586         rte_spinlock_unlock(&cmng->pool_update_sl);
7587         /* Set the statistic memory to the new created pool. */
7588         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7589                 goto set_alarm;
7590         if (pool->raw_hw)
7591                 /* There is a pool query in progress. */
7592                 goto set_alarm;
7593         pool->raw_hw =
7594                 LIST_FIRST(&sh->cmng.free_stat_raws);
7595         if (!pool->raw_hw)
7596                 /* No free counter statistics raw memory. */
7597                 goto set_alarm;
7598         /*
7599          * Identify the counters released between query trigger and query
7600          * handle more efficiently. The counter released in this gap period
7601          * should wait for a new round of query as the new arrived packets
7602          * will not be taken into account.
7603          */
7604         pool->query_gen++;
7605         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7606                                                MLX5_COUNTERS_PER_POOL,
7607                                                NULL, NULL,
7608                                                pool->raw_hw->mem_mng->dm->id,
7609                                                (void *)(uintptr_t)
7610                                                pool->raw_hw->data,
7611                                                sh->devx_comp,
7612                                                (uint64_t)(uintptr_t)pool);
7613         if (ret) {
7614                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7615                         " %d", pool->min_dcs->id);
7616                 pool->raw_hw = NULL;
7617                 goto set_alarm;
7618         }
7619         LIST_REMOVE(pool->raw_hw, next);
7620         sh->cmng.pending_queries++;
7621         pool_index++;
7622         if (pool_index >= n_valid)
7623                 pool_index = 0;
7624 set_alarm:
7625         sh->cmng.pool_index = pool_index;
7626         mlx5_set_query_alarm(sh);
7627 }
7628
7629 /**
7630  * Check and callback event for new aged flow in the counter pool
7631  *
7632  * @param[in] sh
7633  *   Pointer to mlx5_dev_ctx_shared object.
7634  * @param[in] pool
7635  *   Pointer to Current counter pool.
7636  */
7637 static void
7638 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7639                    struct mlx5_flow_counter_pool *pool)
7640 {
7641         struct mlx5_priv *priv;
7642         struct mlx5_flow_counter *cnt;
7643         struct mlx5_age_info *age_info;
7644         struct mlx5_age_param *age_param;
7645         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7646         struct mlx5_counter_stats_raw *prev = pool->raw;
7647         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7648         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7649         uint16_t expected = AGE_CANDIDATE;
7650         uint32_t i;
7651
7652         pool->time_of_last_age_check = curr_time;
7653         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7654                 cnt = MLX5_POOL_GET_CNT(pool, i);
7655                 age_param = MLX5_CNT_TO_AGE(cnt);
7656                 if (__atomic_load_n(&age_param->state,
7657                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7658                         continue;
7659                 if (cur->data[i].hits != prev->data[i].hits) {
7660                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7661                                          __ATOMIC_RELAXED);
7662                         continue;
7663                 }
7664                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7665                                        time_delta,
7666                                        __ATOMIC_RELAXED) <= age_param->timeout)
7667                         continue;
7668                 /**
7669                  * Hold the lock first, or if between the
7670                  * state AGE_TMOUT and tailq operation the
7671                  * release happened, the release procedure
7672                  * may delete a non-existent tailq node.
7673                  */
7674                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7675                 age_info = GET_PORT_AGE_INFO(priv);
7676                 rte_spinlock_lock(&age_info->aged_sl);
7677                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7678                                                 AGE_TMOUT, false,
7679                                                 __ATOMIC_RELAXED,
7680                                                 __ATOMIC_RELAXED)) {
7681                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7682                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7683                 }
7684                 rte_spinlock_unlock(&age_info->aged_sl);
7685         }
7686         mlx5_age_event_prepare(sh);
7687 }
7688
7689 /**
7690  * Handler for the HW respond about ready values from an asynchronous batch
7691  * query. This function is probably called by the host thread.
7692  *
7693  * @param[in] sh
7694  *   The pointer to the shared device context.
7695  * @param[in] async_id
7696  *   The Devx async ID.
7697  * @param[in] status
7698  *   The status of the completion.
7699  */
7700 void
7701 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7702                                   uint64_t async_id, int status)
7703 {
7704         struct mlx5_flow_counter_pool *pool =
7705                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7706         struct mlx5_counter_stats_raw *raw_to_free;
7707         uint8_t query_gen = pool->query_gen ^ 1;
7708         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7709         enum mlx5_counter_type cnt_type =
7710                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7711                                 MLX5_COUNTER_TYPE_ORIGIN;
7712
7713         if (unlikely(status)) {
7714                 raw_to_free = pool->raw_hw;
7715         } else {
7716                 raw_to_free = pool->raw;
7717                 if (pool->is_aged)
7718                         mlx5_flow_aging_check(sh, pool);
7719                 rte_spinlock_lock(&pool->sl);
7720                 pool->raw = pool->raw_hw;
7721                 rte_spinlock_unlock(&pool->sl);
7722                 /* Be sure the new raw counters data is updated in memory. */
7723                 rte_io_wmb();
7724                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7725                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7726                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7727                                      &pool->counters[query_gen], next);
7728                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7729                 }
7730         }
7731         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7732         pool->raw_hw = NULL;
7733         sh->cmng.pending_queries--;
7734 }
7735
7736 static int
7737 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7738                     const struct flow_grp_info *grp_info,
7739                     struct rte_flow_error *error)
7740 {
7741         if (grp_info->transfer && grp_info->external &&
7742             grp_info->fdb_def_rule) {
7743                 if (group == UINT32_MAX)
7744                         return rte_flow_error_set
7745                                                 (error, EINVAL,
7746                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7747                                                  NULL,
7748                                                  "group index not supported");
7749                 *table = group + 1;
7750         } else {
7751                 *table = group;
7752         }
7753         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7754         return 0;
7755 }
7756
7757 /**
7758  * Translate the rte_flow group index to HW table value.
7759  *
7760  * If tunnel offload is disabled, all group ids converted to flow table
7761  * id using the standard method.
7762  * If tunnel offload is enabled, group id can be converted using the
7763  * standard or tunnel conversion method. Group conversion method
7764  * selection depends on flags in `grp_info` parameter:
7765  * - Internal (grp_info.external == 0) groups conversion uses the
7766  *   standard method.
7767  * - Group ids in JUMP action converted with the tunnel conversion.
7768  * - Group id in rule attribute conversion depends on a rule type and
7769  *   group id value:
7770  *   ** non zero group attributes converted with the tunnel method
7771  *   ** zero group attribute in non-tunnel rule is converted using the
7772  *      standard method - there's only one root table
7773  *   ** zero group attribute in steer tunnel rule is converted with the
7774  *      standard method - single root table
7775  *   ** zero group attribute in match tunnel rule is a special OvS
7776  *      case: that value is used for portability reasons. That group
7777  *      id is converted with the tunnel conversion method.
7778  *
7779  * @param[in] dev
7780  *   Port device
7781  * @param[in] tunnel
7782  *   PMD tunnel offload object
7783  * @param[in] group
7784  *   rte_flow group index value.
7785  * @param[out] table
7786  *   HW table value.
7787  * @param[in] grp_info
7788  *   flags used for conversion
7789  * @param[out] error
7790  *   Pointer to error structure.
7791  *
7792  * @return
7793  *   0 on success, a negative errno value otherwise and rte_errno is set.
7794  */
7795 int
7796 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7797                          const struct mlx5_flow_tunnel *tunnel,
7798                          uint32_t group, uint32_t *table,
7799                          const struct flow_grp_info *grp_info,
7800                          struct rte_flow_error *error)
7801 {
7802         int ret;
7803         bool standard_translation;
7804
7805         if (!grp_info->skip_scale && grp_info->external &&
7806             group < MLX5_MAX_TABLES_EXTERNAL)
7807                 group *= MLX5_FLOW_TABLE_FACTOR;
7808         if (is_tunnel_offload_active(dev)) {
7809                 standard_translation = !grp_info->external ||
7810                                         grp_info->std_tbl_fix;
7811         } else {
7812                 standard_translation = true;
7813         }
7814         DRV_LOG(DEBUG,
7815                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7816                 dev->data->port_id, group, grp_info->transfer,
7817                 grp_info->external, grp_info->fdb_def_rule,
7818                 standard_translation ? "STANDARD" : "TUNNEL");
7819         if (standard_translation)
7820                 ret = flow_group_to_table(dev->data->port_id, group, table,
7821                                           grp_info, error);
7822         else
7823                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7824                                                       table, error);
7825
7826         return ret;
7827 }
7828
7829 /**
7830  * Discover availability of metadata reg_c's.
7831  *
7832  * Iteratively use test flows to check availability.
7833  *
7834  * @param[in] dev
7835  *   Pointer to the Ethernet device structure.
7836  *
7837  * @return
7838  *   0 on success, a negative errno value otherwise and rte_errno is set.
7839  */
7840 int
7841 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7842 {
7843         struct mlx5_priv *priv = dev->data->dev_private;
7844         struct mlx5_dev_config *config = &priv->config;
7845         enum modify_reg idx;
7846         int n = 0;
7847
7848         /* reg_c[0] and reg_c[1] are reserved. */
7849         config->flow_mreg_c[n++] = REG_C_0;
7850         config->flow_mreg_c[n++] = REG_C_1;
7851         /* Discover availability of other reg_c's. */
7852         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7853                 struct rte_flow_attr attr = {
7854                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7855                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7856                         .ingress = 1,
7857                 };
7858                 struct rte_flow_item items[] = {
7859                         [0] = {
7860                                 .type = RTE_FLOW_ITEM_TYPE_END,
7861                         },
7862                 };
7863                 struct rte_flow_action actions[] = {
7864                         [0] = {
7865                                 .type = (enum rte_flow_action_type)
7866                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7867                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7868                                         .src = REG_C_1,
7869                                         .dst = idx,
7870                                 },
7871                         },
7872                         [1] = {
7873                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7874                                 .conf = &(struct rte_flow_action_jump){
7875                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7876                                 },
7877                         },
7878                         [2] = {
7879                                 .type = RTE_FLOW_ACTION_TYPE_END,
7880                         },
7881                 };
7882                 uint32_t flow_idx;
7883                 struct rte_flow *flow;
7884                 struct rte_flow_error error;
7885
7886                 if (!config->dv_flow_en)
7887                         break;
7888                 /* Create internal flow, validation skips copy action. */
7889                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7890                                             actions, false, &error);
7891                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7892                                       flow_idx);
7893                 if (!flow)
7894                         continue;
7895                 config->flow_mreg_c[n++] = idx;
7896                 flow_list_destroy(dev, NULL, flow_idx);
7897         }
7898         for (; n < MLX5_MREG_C_NUM; ++n)
7899                 config->flow_mreg_c[n] = REG_NON;
7900         return 0;
7901 }
7902
7903 int
7904 save_dump_file(const uint8_t *data, uint32_t size,
7905         uint32_t type, uint32_t id, void *arg, FILE *file)
7906 {
7907         char line[BUF_SIZE];
7908         uint32_t out = 0;
7909         uint32_t k;
7910         uint32_t actions_num;
7911         struct rte_flow_query_count *count;
7912
7913         memset(line, 0, BUF_SIZE);
7914         switch (type) {
7915         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
7916                 actions_num = *(uint32_t *)(arg);
7917                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,",
7918                                 type, id, actions_num);
7919                 break;
7920         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
7921                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,",
7922                                 type, id);
7923                 break;
7924         case DR_DUMP_REC_TYPE_PMD_COUNTER:
7925                 count = (struct rte_flow_query_count *)arg;
7926                 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type,
7927                                 id, count->hits, count->bytes);
7928                 return 0;
7929         default:
7930                 return -1;
7931         }
7932
7933         for (k = 0; k < size; k++) {
7934                 /* Make sure we do not overrun the line buffer length. */
7935                 if (out >= BUF_SIZE - 4) {
7936                         line[out] = '\0';
7937                         break;
7938                 }
7939                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
7940                                 (data[k]) & 0xff);
7941         }
7942         fprintf(file, "%s\n", line);
7943         return 0;
7944 }
7945
7946 int
7947 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
7948         struct rte_flow_query_count *count, struct rte_flow_error *error)
7949 {
7950         struct rte_flow_action action[2];
7951         enum mlx5_flow_drv_type ftype;
7952         const struct mlx5_flow_driver_ops *fops;
7953
7954         if (!flow) {
7955                 return rte_flow_error_set(error, ENOENT,
7956                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7957                                 NULL,
7958                                 "invalid flow handle");
7959         }
7960         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
7961         action[1].type = RTE_FLOW_ACTION_TYPE_END;
7962         if (flow->counter) {
7963                 memset(count, 0, sizeof(struct rte_flow_query_count));
7964                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
7965                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
7966                                                 ftype < MLX5_FLOW_TYPE_MAX);
7967                 fops = flow_get_drv_ops(ftype);
7968                 return fops->query(dev, flow, action, count, error);
7969         }
7970         return -1;
7971 }
7972
7973 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7974 /**
7975  * Dump flow ipool data to file
7976  *
7977  * @param[in] dev
7978  *   The pointer to Ethernet device.
7979  * @param[in] file
7980  *   A pointer to a file for output.
7981  * @param[out] error
7982  *   Perform verbose error reporting if not NULL. PMDs initialize this
7983  *   structure in case of error only.
7984  * @return
7985  *   0 on success, a negative value otherwise.
7986  */
7987 int
7988 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
7989         struct rte_flow *flow, FILE *file,
7990         struct rte_flow_error *error)
7991 {
7992         struct mlx5_priv *priv = dev->data->dev_private;
7993         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
7994         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
7995         uint32_t handle_idx;
7996         struct mlx5_flow_handle *dh;
7997         struct rte_flow_query_count count;
7998         uint32_t actions_num;
7999         const uint8_t *data;
8000         size_t size;
8001         uint32_t id;
8002         uint32_t type;
8003
8004         if (!flow) {
8005                 return rte_flow_error_set(error, ENOENT,
8006                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8007                         NULL,
8008                         "invalid flow handle");
8009         }
8010         handle_idx = flow->dev_handles;
8011         while (handle_idx) {
8012                 dh = mlx5_ipool_get(priv->sh->ipool
8013                         [MLX5_IPOOL_MLX5_FLOW], handle_idx);
8014                 if (!dh)
8015                         continue;
8016                 handle_idx = dh->next.next;
8017                 id = (uint32_t)(uintptr_t)dh->drv_flow;
8018
8019                 /* query counter */
8020                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
8021                 if (!mlx5_flow_query_counter(dev, flow, &count, error))
8022                         save_dump_file(NULL, 0, type,
8023                                         id, (void *)&count, file);
8024
8025                 /* Get modify_hdr and encap_decap buf from ipools. */
8026                 encap_decap = NULL;
8027                 modify_hdr = dh->dvh.modify_hdr;
8028
8029                 if (dh->dvh.rix_encap_decap) {
8030                         encap_decap = mlx5_ipool_get(priv->sh->ipool
8031                                                 [MLX5_IPOOL_DECAP_ENCAP],
8032                                                 dh->dvh.rix_encap_decap);
8033                 }
8034                 if (modify_hdr) {
8035                         data = (const uint8_t *)modify_hdr->actions;
8036                         size = (size_t)(modify_hdr->actions_num) * 8;
8037                         actions_num = modify_hdr->actions_num;
8038                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
8039                         save_dump_file(data, size, type, id,
8040                                         (void *)(&actions_num), file);
8041                 }
8042                 if (encap_decap) {
8043                         data = encap_decap->buf;
8044                         size = encap_decap->size;
8045                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
8046                         save_dump_file(data, size, type,
8047                                                 id, NULL, file);
8048                 }
8049         }
8050         return 0;
8051 }
8052 #endif
8053
8054 /**
8055  * Dump flow raw hw data to file
8056  *
8057  * @param[in] dev
8058  *    The pointer to Ethernet device.
8059  * @param[in] file
8060  *   A pointer to a file for output.
8061  * @param[out] error
8062  *   Perform verbose error reporting if not NULL. PMDs initialize this
8063  *   structure in case of error only.
8064  * @return
8065  *   0 on success, a nagative value otherwise.
8066  */
8067 int
8068 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
8069                    FILE *file,
8070                    struct rte_flow_error *error __rte_unused)
8071 {
8072         struct mlx5_priv *priv = dev->data->dev_private;
8073         struct mlx5_dev_ctx_shared *sh = priv->sh;
8074         uint32_t handle_idx;
8075         int ret;
8076         struct mlx5_flow_handle *dh;
8077         struct rte_flow *flow;
8078 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8079         uint32_t idx;
8080 #endif
8081
8082         if (!priv->config.dv_flow_en) {
8083                 if (fputs("device dv flow disabled\n", file) <= 0)
8084                         return -errno;
8085                 return -ENOTSUP;
8086         }
8087
8088         /* dump all */
8089         if (!flow_idx) {
8090 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8091                 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
8092                                                 priv->flows, idx, flow, next)
8093                         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8094 #endif
8095                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
8096                                         sh->rx_domain,
8097                                         sh->tx_domain, file);
8098         }
8099         /* dump one */
8100         flow = mlx5_ipool_get(priv->sh->ipool
8101                         [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
8102         if (!flow)
8103                 return -ENOENT;
8104
8105 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8106         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8107 #endif
8108         handle_idx = flow->dev_handles;
8109         while (handle_idx) {
8110                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8111                                 handle_idx);
8112                 if (!dh)
8113                         return -ENOENT;
8114                 if (dh->drv_flow) {
8115                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
8116                                         file);
8117                         if (ret)
8118                                 return -ENOENT;
8119                 }
8120                 handle_idx = dh->next.next;
8121         }
8122         return 0;
8123 }
8124
8125 /**
8126  * Get aged-out flows.
8127  *
8128  * @param[in] dev
8129  *   Pointer to the Ethernet device structure.
8130  * @param[in] context
8131  *   The address of an array of pointers to the aged-out flows contexts.
8132  * @param[in] nb_countexts
8133  *   The length of context array pointers.
8134  * @param[out] error
8135  *   Perform verbose error reporting if not NULL. Initialized in case of
8136  *   error only.
8137  *
8138  * @return
8139  *   how many contexts get in success, otherwise negative errno value.
8140  *   if nb_contexts is 0, return the amount of all aged contexts.
8141  *   if nb_contexts is not 0 , return the amount of aged flows reported
8142  *   in the context array.
8143  */
8144 int
8145 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
8146                         uint32_t nb_contexts, struct rte_flow_error *error)
8147 {
8148         const struct mlx5_flow_driver_ops *fops;
8149         struct rte_flow_attr attr = { .transfer = 0 };
8150
8151         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8152                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8153                 return fops->get_aged_flows(dev, contexts, nb_contexts,
8154                                                     error);
8155         }
8156         DRV_LOG(ERR,
8157                 "port %u get aged flows is not supported.",
8158                  dev->data->port_id);
8159         return -ENOTSUP;
8160 }
8161
8162 /* Wrapper for driver action_validate op callback */
8163 static int
8164 flow_drv_action_validate(struct rte_eth_dev *dev,
8165                          const struct rte_flow_indir_action_conf *conf,
8166                          const struct rte_flow_action *action,
8167                          const struct mlx5_flow_driver_ops *fops,
8168                          struct rte_flow_error *error)
8169 {
8170         static const char err_msg[] = "indirect action validation unsupported";
8171
8172         if (!fops->action_validate) {
8173                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8174                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8175                                    NULL, err_msg);
8176                 return -rte_errno;
8177         }
8178         return fops->action_validate(dev, conf, action, error);
8179 }
8180
8181 /**
8182  * Destroys the shared action by handle.
8183  *
8184  * @param dev
8185  *   Pointer to Ethernet device structure.
8186  * @param[in] handle
8187  *   Handle for the indirect action object to be destroyed.
8188  * @param[out] error
8189  *   Perform verbose error reporting if not NULL. PMDs initialize this
8190  *   structure in case of error only.
8191  *
8192  * @return
8193  *   0 on success, a negative errno value otherwise and rte_errno is set.
8194  *
8195  * @note: wrapper for driver action_create op callback.
8196  */
8197 static int
8198 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
8199                            struct rte_flow_action_handle *handle,
8200                            struct rte_flow_error *error)
8201 {
8202         static const char err_msg[] = "indirect action destruction unsupported";
8203         struct rte_flow_attr attr = { .transfer = 0 };
8204         const struct mlx5_flow_driver_ops *fops =
8205                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8206
8207         if (!fops->action_destroy) {
8208                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8209                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8210                                    NULL, err_msg);
8211                 return -rte_errno;
8212         }
8213         return fops->action_destroy(dev, handle, error);
8214 }
8215
8216 /* Wrapper for driver action_destroy op callback */
8217 static int
8218 flow_drv_action_update(struct rte_eth_dev *dev,
8219                        struct rte_flow_action_handle *handle,
8220                        const void *update,
8221                        const struct mlx5_flow_driver_ops *fops,
8222                        struct rte_flow_error *error)
8223 {
8224         static const char err_msg[] = "indirect action update unsupported";
8225
8226         if (!fops->action_update) {
8227                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8228                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8229                                    NULL, err_msg);
8230                 return -rte_errno;
8231         }
8232         return fops->action_update(dev, handle, update, error);
8233 }
8234
8235 /* Wrapper for driver action_destroy op callback */
8236 static int
8237 flow_drv_action_query(struct rte_eth_dev *dev,
8238                       const struct rte_flow_action_handle *handle,
8239                       void *data,
8240                       const struct mlx5_flow_driver_ops *fops,
8241                       struct rte_flow_error *error)
8242 {
8243         static const char err_msg[] = "indirect action query unsupported";
8244
8245         if (!fops->action_query) {
8246                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8247                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8248                                    NULL, err_msg);
8249                 return -rte_errno;
8250         }
8251         return fops->action_query(dev, handle, data, error);
8252 }
8253
8254 /**
8255  * Create indirect action for reuse in multiple flow rules.
8256  *
8257  * @param dev
8258  *   Pointer to Ethernet device structure.
8259  * @param conf
8260  *   Pointer to indirect action object configuration.
8261  * @param[in] action
8262  *   Action configuration for indirect action object creation.
8263  * @param[out] error
8264  *   Perform verbose error reporting if not NULL. PMDs initialize this
8265  *   structure in case of error only.
8266  * @return
8267  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8268  */
8269 static struct rte_flow_action_handle *
8270 mlx5_action_handle_create(struct rte_eth_dev *dev,
8271                           const struct rte_flow_indir_action_conf *conf,
8272                           const struct rte_flow_action *action,
8273                           struct rte_flow_error *error)
8274 {
8275         static const char err_msg[] = "indirect action creation unsupported";
8276         struct rte_flow_attr attr = { .transfer = 0 };
8277         const struct mlx5_flow_driver_ops *fops =
8278                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8279
8280         if (flow_drv_action_validate(dev, conf, action, fops, error))
8281                 return NULL;
8282         if (!fops->action_create) {
8283                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8284                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8285                                    NULL, err_msg);
8286                 return NULL;
8287         }
8288         return fops->action_create(dev, conf, action, error);
8289 }
8290
8291 /**
8292  * Updates inplace the indirect action configuration pointed by *handle*
8293  * with the configuration provided as *update* argument.
8294  * The update of the indirect action configuration effects all flow rules
8295  * reusing the action via handle.
8296  *
8297  * @param dev
8298  *   Pointer to Ethernet device structure.
8299  * @param[in] handle
8300  *   Handle for the indirect action to be updated.
8301  * @param[in] update
8302  *   Action specification used to modify the action pointed by handle.
8303  *   *update* could be of same type with the action pointed by the *handle*
8304  *   handle argument, or some other structures like a wrapper, depending on
8305  *   the indirect action type.
8306  * @param[out] error
8307  *   Perform verbose error reporting if not NULL. PMDs initialize this
8308  *   structure in case of error only.
8309  *
8310  * @return
8311  *   0 on success, a negative errno value otherwise and rte_errno is set.
8312  */
8313 static int
8314 mlx5_action_handle_update(struct rte_eth_dev *dev,
8315                 struct rte_flow_action_handle *handle,
8316                 const void *update,
8317                 struct rte_flow_error *error)
8318 {
8319         struct rte_flow_attr attr = { .transfer = 0 };
8320         const struct mlx5_flow_driver_ops *fops =
8321                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8322         int ret;
8323
8324         ret = flow_drv_action_validate(dev, NULL,
8325                         (const struct rte_flow_action *)update, fops, error);
8326         if (ret)
8327                 return ret;
8328         return flow_drv_action_update(dev, handle, update, fops,
8329                                       error);
8330 }
8331
8332 /**
8333  * Query the indirect action by handle.
8334  *
8335  * This function allows retrieving action-specific data such as counters.
8336  * Data is gathered by special action which may be present/referenced in
8337  * more than one flow rule definition.
8338  *
8339  * see @RTE_FLOW_ACTION_TYPE_COUNT
8340  *
8341  * @param dev
8342  *   Pointer to Ethernet device structure.
8343  * @param[in] handle
8344  *   Handle for the indirect action to query.
8345  * @param[in, out] data
8346  *   Pointer to storage for the associated query data type.
8347  * @param[out] error
8348  *   Perform verbose error reporting if not NULL. PMDs initialize this
8349  *   structure in case of error only.
8350  *
8351  * @return
8352  *   0 on success, a negative errno value otherwise and rte_errno is set.
8353  */
8354 static int
8355 mlx5_action_handle_query(struct rte_eth_dev *dev,
8356                          const struct rte_flow_action_handle *handle,
8357                          void *data,
8358                          struct rte_flow_error *error)
8359 {
8360         struct rte_flow_attr attr = { .transfer = 0 };
8361         const struct mlx5_flow_driver_ops *fops =
8362                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8363
8364         return flow_drv_action_query(dev, handle, data, fops, error);
8365 }
8366
8367 /**
8368  * Destroy all indirect actions (shared RSS).
8369  *
8370  * @param dev
8371  *   Pointer to Ethernet device.
8372  *
8373  * @return
8374  *   0 on success, a negative errno value otherwise and rte_errno is set.
8375  */
8376 int
8377 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8378 {
8379         struct rte_flow_error error;
8380         struct mlx5_priv *priv = dev->data->dev_private;
8381         struct mlx5_shared_action_rss *shared_rss;
8382         int ret = 0;
8383         uint32_t idx;
8384
8385         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8386                       priv->rss_shared_actions, idx, shared_rss, next) {
8387                 ret |= mlx5_action_handle_destroy(dev,
8388                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8389         }
8390         return ret;
8391 }
8392
8393 #ifndef HAVE_MLX5DV_DR
8394 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8395 #else
8396 #define MLX5_DOMAIN_SYNC_FLOW \
8397         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8398 #endif
8399
8400 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8401 {
8402         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8403         const struct mlx5_flow_driver_ops *fops;
8404         int ret;
8405         struct rte_flow_attr attr = { .transfer = 0 };
8406
8407         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8408         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8409         if (ret > 0)
8410                 ret = -ret;
8411         return ret;
8412 }
8413
8414 const struct mlx5_flow_tunnel *
8415 mlx5_get_tof(const struct rte_flow_item *item,
8416              const struct rte_flow_action *action,
8417              enum mlx5_tof_rule_type *rule_type)
8418 {
8419         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8420                 if (item->type == (typeof(item->type))
8421                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8422                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8423                         return flow_items_to_tunnel(item);
8424                 }
8425         }
8426         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8427                 if (action->type == (typeof(action->type))
8428                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8429                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8430                         return flow_actions_to_tunnel(action);
8431                 }
8432         }
8433         return NULL;
8434 }
8435
8436 /**
8437  * tunnel offload functionalilty is defined for DV environment only
8438  */
8439 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8440 __extension__
8441 union tunnel_offload_mark {
8442         uint32_t val;
8443         struct {
8444                 uint32_t app_reserve:8;
8445                 uint32_t table_id:15;
8446                 uint32_t transfer:1;
8447                 uint32_t _unused_:8;
8448         };
8449 };
8450
8451 static bool
8452 mlx5_access_tunnel_offload_db
8453         (struct rte_eth_dev *dev,
8454          bool (*match)(struct rte_eth_dev *,
8455                        struct mlx5_flow_tunnel *, const void *),
8456          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8457          void (*miss)(struct rte_eth_dev *, void *),
8458          void *ctx, bool lock_op);
8459
8460 static int
8461 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8462                              struct rte_flow *flow,
8463                              const struct rte_flow_attr *attr,
8464                              const struct rte_flow_action *app_actions,
8465                              uint32_t flow_idx,
8466                              const struct mlx5_flow_tunnel *tunnel,
8467                              struct tunnel_default_miss_ctx *ctx,
8468                              struct rte_flow_error *error)
8469 {
8470         struct mlx5_priv *priv = dev->data->dev_private;
8471         struct mlx5_flow *dev_flow;
8472         struct rte_flow_attr miss_attr = *attr;
8473         const struct rte_flow_item miss_items[2] = {
8474                 {
8475                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8476                         .spec = NULL,
8477                         .last = NULL,
8478                         .mask = NULL
8479                 },
8480                 {
8481                         .type = RTE_FLOW_ITEM_TYPE_END,
8482                         .spec = NULL,
8483                         .last = NULL,
8484                         .mask = NULL
8485                 }
8486         };
8487         union tunnel_offload_mark mark_id;
8488         struct rte_flow_action_mark miss_mark;
8489         struct rte_flow_action miss_actions[3] = {
8490                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8491                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8492         };
8493         const struct rte_flow_action_jump *jump_data;
8494         uint32_t i, flow_table = 0; /* prevent compilation warning */
8495         struct flow_grp_info grp_info = {
8496                 .external = 1,
8497                 .transfer = attr->transfer,
8498                 .fdb_def_rule = !!priv->fdb_def_rule,
8499                 .std_tbl_fix = 0,
8500         };
8501         int ret;
8502
8503         if (!attr->transfer) {
8504                 uint32_t q_size;
8505
8506                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8507                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8508                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8509                                          0, SOCKET_ID_ANY);
8510                 if (!ctx->queue)
8511                         return rte_flow_error_set
8512                                 (error, ENOMEM,
8513                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8514                                 NULL, "invalid default miss RSS");
8515                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8516                 ctx->action_rss.level = 0,
8517                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8518                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8519                 ctx->action_rss.queue_num = priv->reta_idx_n,
8520                 ctx->action_rss.key = priv->rss_conf.rss_key,
8521                 ctx->action_rss.queue = ctx->queue;
8522                 if (!priv->reta_idx_n || !priv->rxqs_n)
8523                         return rte_flow_error_set
8524                                 (error, EINVAL,
8525                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8526                                 NULL, "invalid port configuration");
8527                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8528                         ctx->action_rss.types = 0;
8529                 for (i = 0; i != priv->reta_idx_n; ++i)
8530                         ctx->queue[i] = (*priv->reta_idx)[i];
8531         } else {
8532                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8533                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8534         }
8535         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8536         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8537         jump_data = app_actions->conf;
8538         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8539         miss_attr.group = jump_data->group;
8540         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8541                                        &flow_table, &grp_info, error);
8542         if (ret)
8543                 return rte_flow_error_set(error, EINVAL,
8544                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8545                                           NULL, "invalid tunnel id");
8546         mark_id.app_reserve = 0;
8547         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8548         mark_id.transfer = !!attr->transfer;
8549         mark_id._unused_ = 0;
8550         miss_mark.id = mark_id.val;
8551         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8552                                     miss_items, miss_actions, flow_idx, error);
8553         if (!dev_flow)
8554                 return -rte_errno;
8555         dev_flow->flow = flow;
8556         dev_flow->external = true;
8557         dev_flow->tunnel = tunnel;
8558         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8559         /* Subflow object was created, we must include one in the list. */
8560         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8561                       dev_flow->handle, next);
8562         DRV_LOG(DEBUG,
8563                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8564                 dev->data->port_id, tunnel->app_tunnel.type,
8565                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8566         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8567                                   miss_actions, error);
8568         if (!ret)
8569                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8570                                                   error);
8571
8572         return ret;
8573 }
8574
8575 static const struct mlx5_flow_tbl_data_entry  *
8576 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8577 {
8578         struct mlx5_priv *priv = dev->data->dev_private;
8579         struct mlx5_dev_ctx_shared *sh = priv->sh;
8580         struct mlx5_hlist_entry *he;
8581         union tunnel_offload_mark mbits = { .val = mark };
8582         union mlx5_flow_tbl_key table_key = {
8583                 {
8584                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8585                         .id = 0,
8586                         .reserved = 0,
8587                         .dummy = 0,
8588                         .is_fdb = !!mbits.transfer,
8589                         .is_egress = 0,
8590                 }
8591         };
8592         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
8593         return he ?
8594                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8595 }
8596
8597 static void
8598 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
8599                                    struct mlx5_hlist_entry *entry)
8600 {
8601         struct mlx5_dev_ctx_shared *sh = list->ctx;
8602         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8603
8604         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8605                         tunnel_flow_tbl_to_id(tte->flow_table));
8606         mlx5_free(tte);
8607 }
8608
8609 static int
8610 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
8611                                   struct mlx5_hlist_entry *entry,
8612                                   uint64_t key, void *cb_ctx __rte_unused)
8613 {
8614         union tunnel_tbl_key tbl = {
8615                 .val = key,
8616         };
8617         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8618
8619         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8620 }
8621
8622 static struct mlx5_hlist_entry *
8623 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
8624                                    void *ctx __rte_unused)
8625 {
8626         struct mlx5_dev_ctx_shared *sh = list->ctx;
8627         struct tunnel_tbl_entry *tte;
8628         union tunnel_tbl_key tbl = {
8629                 .val = key,
8630         };
8631
8632         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8633                           sizeof(*tte), 0,
8634                           SOCKET_ID_ANY);
8635         if (!tte)
8636                 goto err;
8637         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8638                           &tte->flow_table);
8639         if (tte->flow_table >= MLX5_MAX_TABLES) {
8640                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8641                         tte->flow_table);
8642                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8643                                 tte->flow_table);
8644                 goto err;
8645         } else if (!tte->flow_table) {
8646                 goto err;
8647         }
8648         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8649         tte->tunnel_id = tbl.tunnel_id;
8650         tte->group = tbl.group;
8651         return &tte->hash;
8652 err:
8653         if (tte)
8654                 mlx5_free(tte);
8655         return NULL;
8656 }
8657
8658 static uint32_t
8659 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8660                                 const struct mlx5_flow_tunnel *tunnel,
8661                                 uint32_t group, uint32_t *table,
8662                                 struct rte_flow_error *error)
8663 {
8664         struct mlx5_hlist_entry *he;
8665         struct tunnel_tbl_entry *tte;
8666         union tunnel_tbl_key key = {
8667                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8668                 .group = group
8669         };
8670         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8671         struct mlx5_hlist *group_hash;
8672
8673         group_hash = tunnel ? tunnel->groups : thub->groups;
8674         he = mlx5_hlist_register(group_hash, key.val, NULL);
8675         if (!he)
8676                 return rte_flow_error_set(error, EINVAL,
8677                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8678                                           NULL,
8679                                           "tunnel group index not supported");
8680         tte = container_of(he, typeof(*tte), hash);
8681         *table = tte->flow_table;
8682         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8683                 dev->data->port_id, key.tunnel_id, group, *table);
8684         return 0;
8685 }
8686
8687 static void
8688 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8689                       struct mlx5_flow_tunnel *tunnel)
8690 {
8691         struct mlx5_priv *priv = dev->data->dev_private;
8692         struct mlx5_indexed_pool *ipool;
8693
8694         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8695                 dev->data->port_id, tunnel->tunnel_id);
8696         LIST_REMOVE(tunnel, chain);
8697         mlx5_hlist_destroy(tunnel->groups);
8698         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8699         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8700 }
8701
8702 static bool
8703 mlx5_access_tunnel_offload_db
8704         (struct rte_eth_dev *dev,
8705          bool (*match)(struct rte_eth_dev *,
8706                        struct mlx5_flow_tunnel *, const void *),
8707          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8708          void (*miss)(struct rte_eth_dev *, void *),
8709          void *ctx, bool lock_op)
8710 {
8711         bool verdict = false;
8712         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8713         struct mlx5_flow_tunnel *tunnel;
8714
8715         rte_spinlock_lock(&thub->sl);
8716         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8717                 verdict = match(dev, tunnel, (const void *)ctx);
8718                 if (verdict)
8719                         break;
8720         }
8721         if (!lock_op)
8722                 rte_spinlock_unlock(&thub->sl);
8723         if (verdict && hit)
8724                 hit(dev, tunnel, ctx);
8725         if (!verdict && miss)
8726                 miss(dev, ctx);
8727         if (lock_op)
8728                 rte_spinlock_unlock(&thub->sl);
8729
8730         return verdict;
8731 }
8732
8733 struct tunnel_db_find_tunnel_id_ctx {
8734         uint32_t tunnel_id;
8735         struct mlx5_flow_tunnel *tunnel;
8736 };
8737
8738 static bool
8739 find_tunnel_id_match(struct rte_eth_dev *dev,
8740                      struct mlx5_flow_tunnel *tunnel, const void *x)
8741 {
8742         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8743
8744         RTE_SET_USED(dev);
8745         return tunnel->tunnel_id == ctx->tunnel_id;
8746 }
8747
8748 static void
8749 find_tunnel_id_hit(struct rte_eth_dev *dev,
8750                    struct mlx5_flow_tunnel *tunnel, void *x)
8751 {
8752         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8753         RTE_SET_USED(dev);
8754         ctx->tunnel = tunnel;
8755 }
8756
8757 static struct mlx5_flow_tunnel *
8758 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8759 {
8760         struct tunnel_db_find_tunnel_id_ctx ctx = {
8761                 .tunnel_id = id,
8762         };
8763
8764         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8765                                       find_tunnel_id_hit, NULL, &ctx, true);
8766
8767         return ctx.tunnel;
8768 }
8769
8770 static struct mlx5_flow_tunnel *
8771 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8772                           const struct rte_flow_tunnel *app_tunnel)
8773 {
8774         struct mlx5_priv *priv = dev->data->dev_private;
8775         struct mlx5_indexed_pool *ipool;
8776         struct mlx5_flow_tunnel *tunnel;
8777         uint32_t id;
8778
8779         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8780         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8781         if (!tunnel)
8782                 return NULL;
8783         if (id >= MLX5_MAX_TUNNELS) {
8784                 mlx5_ipool_free(ipool, id);
8785                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8786                 return NULL;
8787         }
8788         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
8789                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8790                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8791                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
8792         if (!tunnel->groups) {
8793                 mlx5_ipool_free(ipool, id);
8794                 return NULL;
8795         }
8796         tunnel->groups->ctx = priv->sh;
8797         /* initiate new PMD tunnel */
8798         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8799         tunnel->tunnel_id = id;
8800         tunnel->action.type = (typeof(tunnel->action.type))
8801                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8802         tunnel->action.conf = tunnel;
8803         tunnel->item.type = (typeof(tunnel->item.type))
8804                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8805         tunnel->item.spec = tunnel;
8806         tunnel->item.last = NULL;
8807         tunnel->item.mask = NULL;
8808
8809         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8810                 dev->data->port_id, tunnel->tunnel_id);
8811
8812         return tunnel;
8813 }
8814
8815 struct tunnel_db_get_tunnel_ctx {
8816         const struct rte_flow_tunnel *app_tunnel;
8817         struct mlx5_flow_tunnel *tunnel;
8818 };
8819
8820 static bool get_tunnel_match(struct rte_eth_dev *dev,
8821                              struct mlx5_flow_tunnel *tunnel, const void *x)
8822 {
8823         const struct tunnel_db_get_tunnel_ctx *ctx = x;
8824
8825         RTE_SET_USED(dev);
8826         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
8827                        sizeof(*ctx->app_tunnel));
8828 }
8829
8830 static void get_tunnel_hit(struct rte_eth_dev *dev,
8831                            struct mlx5_flow_tunnel *tunnel, void *x)
8832 {
8833         /* called under tunnel spinlock protection */
8834         struct tunnel_db_get_tunnel_ctx *ctx = x;
8835
8836         RTE_SET_USED(dev);
8837         tunnel->refctn++;
8838         ctx->tunnel = tunnel;
8839 }
8840
8841 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
8842 {
8843         /* called under tunnel spinlock protection */
8844         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8845         struct tunnel_db_get_tunnel_ctx *ctx = x;
8846
8847         rte_spinlock_unlock(&thub->sl);
8848         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
8849         rte_spinlock_lock(&thub->sl);
8850         if (ctx->tunnel) {
8851                 ctx->tunnel->refctn = 1;
8852                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
8853         }
8854 }
8855
8856
8857 static int
8858 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8859                      const struct rte_flow_tunnel *app_tunnel,
8860                      struct mlx5_flow_tunnel **tunnel)
8861 {
8862         struct tunnel_db_get_tunnel_ctx ctx = {
8863                 .app_tunnel = app_tunnel,
8864         };
8865
8866         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
8867                                       get_tunnel_miss, &ctx, true);
8868         *tunnel = ctx.tunnel;
8869         return ctx.tunnel ? 0 : -ENOMEM;
8870 }
8871
8872 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8873 {
8874         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8875
8876         if (!thub)
8877                 return;
8878         if (!LIST_EMPTY(&thub->tunnels))
8879                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
8880         mlx5_hlist_destroy(thub->groups);
8881         mlx5_free(thub);
8882 }
8883
8884 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8885 {
8886         int err;
8887         struct mlx5_flow_tunnel_hub *thub;
8888
8889         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8890                            0, SOCKET_ID_ANY);
8891         if (!thub)
8892                 return -ENOMEM;
8893         LIST_INIT(&thub->tunnels);
8894         rte_spinlock_init(&thub->sl);
8895         thub->groups = mlx5_hlist_create("flow groups",
8896                                          rte_align32pow2(MLX5_MAX_TABLES), 0,
8897                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
8898                                          mlx5_flow_tunnel_grp2tbl_match_cb,
8899                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
8900         if (!thub->groups) {
8901                 err = -rte_errno;
8902                 goto err;
8903         }
8904         thub->groups->ctx = sh;
8905         sh->tunnel_hub = thub;
8906
8907         return 0;
8908
8909 err:
8910         if (thub->groups)
8911                 mlx5_hlist_destroy(thub->groups);
8912         if (thub)
8913                 mlx5_free(thub);
8914         return err;
8915 }
8916
8917 static inline bool
8918 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
8919                           struct rte_flow_tunnel *tunnel,
8920                           const char *err_msg)
8921 {
8922         err_msg = NULL;
8923         if (!is_tunnel_offload_active(dev)) {
8924                 err_msg = "tunnel offload was not activated";
8925                 goto out;
8926         } else if (!tunnel) {
8927                 err_msg = "no application tunnel";
8928                 goto out;
8929         }
8930
8931         switch (tunnel->type) {
8932         default:
8933                 err_msg = "unsupported tunnel type";
8934                 goto out;
8935         case RTE_FLOW_ITEM_TYPE_VXLAN:
8936                 break;
8937         }
8938
8939 out:
8940         return !err_msg;
8941 }
8942
8943 static int
8944 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
8945                     struct rte_flow_tunnel *app_tunnel,
8946                     struct rte_flow_action **actions,
8947                     uint32_t *num_of_actions,
8948                     struct rte_flow_error *error)
8949 {
8950         int ret;
8951         struct mlx5_flow_tunnel *tunnel;
8952         const char *err_msg = NULL;
8953         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8954
8955         if (!verdict)
8956                 return rte_flow_error_set(error, EINVAL,
8957                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8958                                           err_msg);
8959         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8960         if (ret < 0) {
8961                 return rte_flow_error_set(error, ret,
8962                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8963                                           "failed to initialize pmd tunnel");
8964         }
8965         *actions = &tunnel->action;
8966         *num_of_actions = 1;
8967         return 0;
8968 }
8969
8970 static int
8971 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
8972                        struct rte_flow_tunnel *app_tunnel,
8973                        struct rte_flow_item **items,
8974                        uint32_t *num_of_items,
8975                        struct rte_flow_error *error)
8976 {
8977         int ret;
8978         struct mlx5_flow_tunnel *tunnel;
8979         const char *err_msg = NULL;
8980         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8981
8982         if (!verdict)
8983                 return rte_flow_error_set(error, EINVAL,
8984                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8985                                           err_msg);
8986         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8987         if (ret < 0) {
8988                 return rte_flow_error_set(error, ret,
8989                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8990                                           "failed to initialize pmd tunnel");
8991         }
8992         *items = &tunnel->item;
8993         *num_of_items = 1;
8994         return 0;
8995 }
8996
8997 struct tunnel_db_element_release_ctx {
8998         struct rte_flow_item *items;
8999         struct rte_flow_action *actions;
9000         uint32_t num_elements;
9001         struct rte_flow_error *error;
9002         int ret;
9003 };
9004
9005 static bool
9006 tunnel_element_release_match(struct rte_eth_dev *dev,
9007                              struct mlx5_flow_tunnel *tunnel, const void *x)
9008 {
9009         const struct tunnel_db_element_release_ctx *ctx = x;
9010
9011         RTE_SET_USED(dev);
9012         if (ctx->num_elements != 1)
9013                 return false;
9014         else if (ctx->items)
9015                 return ctx->items == &tunnel->item;
9016         else if (ctx->actions)
9017                 return ctx->actions == &tunnel->action;
9018
9019         return false;
9020 }
9021
9022 static void
9023 tunnel_element_release_hit(struct rte_eth_dev *dev,
9024                            struct mlx5_flow_tunnel *tunnel, void *x)
9025 {
9026         struct tunnel_db_element_release_ctx *ctx = x;
9027         ctx->ret = 0;
9028         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
9029                 mlx5_flow_tunnel_free(dev, tunnel);
9030 }
9031
9032 static void
9033 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
9034 {
9035         struct tunnel_db_element_release_ctx *ctx = x;
9036         RTE_SET_USED(dev);
9037         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
9038                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9039                                       "invalid argument");
9040 }
9041
9042 static int
9043 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
9044                        struct rte_flow_item *pmd_items,
9045                        uint32_t num_items, struct rte_flow_error *err)
9046 {
9047         struct tunnel_db_element_release_ctx ctx = {
9048                 .items = pmd_items,
9049                 .actions = NULL,
9050                 .num_elements = num_items,
9051                 .error = err,
9052         };
9053
9054         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9055                                       tunnel_element_release_hit,
9056                                       tunnel_element_release_miss, &ctx, false);
9057
9058         return ctx.ret;
9059 }
9060
9061 static int
9062 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
9063                          struct rte_flow_action *pmd_actions,
9064                          uint32_t num_actions, struct rte_flow_error *err)
9065 {
9066         struct tunnel_db_element_release_ctx ctx = {
9067                 .items = NULL,
9068                 .actions = pmd_actions,
9069                 .num_elements = num_actions,
9070                 .error = err,
9071         };
9072
9073         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9074                                       tunnel_element_release_hit,
9075                                       tunnel_element_release_miss, &ctx, false);
9076
9077         return ctx.ret;
9078 }
9079
9080 static int
9081 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
9082                                   struct rte_mbuf *m,
9083                                   struct rte_flow_restore_info *info,
9084                                   struct rte_flow_error *err)
9085 {
9086         uint64_t ol_flags = m->ol_flags;
9087         const struct mlx5_flow_tbl_data_entry *tble;
9088         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
9089
9090         if (!is_tunnel_offload_active(dev)) {
9091                 info->flags = 0;
9092                 return 0;
9093         }
9094
9095         if ((ol_flags & mask) != mask)
9096                 goto err;
9097         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
9098         if (!tble) {
9099                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
9100                         dev->data->port_id, m->hash.fdir.hi);
9101                 goto err;
9102         }
9103         MLX5_ASSERT(tble->tunnel);
9104         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
9105         info->group_id = tble->group_id;
9106         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
9107                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
9108                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
9109
9110         return 0;
9111
9112 err:
9113         return rte_flow_error_set(err, EINVAL,
9114                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9115                                   "failed to get restore info");
9116 }
9117
9118 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
9119 static int
9120 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
9121                            __rte_unused struct rte_flow_tunnel *app_tunnel,
9122                            __rte_unused struct rte_flow_action **actions,
9123                            __rte_unused uint32_t *num_of_actions,
9124                            __rte_unused struct rte_flow_error *error)
9125 {
9126         return -ENOTSUP;
9127 }
9128
9129 static int
9130 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
9131                        __rte_unused struct rte_flow_tunnel *app_tunnel,
9132                        __rte_unused struct rte_flow_item **items,
9133                        __rte_unused uint32_t *num_of_items,
9134                        __rte_unused struct rte_flow_error *error)
9135 {
9136         return -ENOTSUP;
9137 }
9138
9139 static int
9140 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
9141                               __rte_unused struct rte_flow_item *pmd_items,
9142                               __rte_unused uint32_t num_items,
9143                               __rte_unused struct rte_flow_error *err)
9144 {
9145         return -ENOTSUP;
9146 }
9147
9148 static int
9149 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
9150                                 __rte_unused struct rte_flow_action *pmd_action,
9151                                 __rte_unused uint32_t num_actions,
9152                                 __rte_unused struct rte_flow_error *err)
9153 {
9154         return -ENOTSUP;
9155 }
9156
9157 static int
9158 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
9159                                   __rte_unused struct rte_mbuf *m,
9160                                   __rte_unused struct rte_flow_restore_info *i,
9161                                   __rte_unused struct rte_flow_error *err)
9162 {
9163         return -ENOTSUP;
9164 }
9165
9166 static int
9167 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
9168                              __rte_unused struct rte_flow *flow,
9169                              __rte_unused const struct rte_flow_attr *attr,
9170                              __rte_unused const struct rte_flow_action *actions,
9171                              __rte_unused uint32_t flow_idx,
9172                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
9173                              __rte_unused struct tunnel_default_miss_ctx *ctx,
9174                              __rte_unused struct rte_flow_error *error)
9175 {
9176         return -ENOTSUP;
9177 }
9178
9179 static struct mlx5_flow_tunnel *
9180 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
9181                     __rte_unused uint32_t id)
9182 {
9183         return NULL;
9184 }
9185
9186 static void
9187 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
9188                       __rte_unused struct mlx5_flow_tunnel *tunnel)
9189 {
9190 }
9191
9192 static uint32_t
9193 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
9194                                 __rte_unused const struct mlx5_flow_tunnel *t,
9195                                 __rte_unused uint32_t group,
9196                                 __rte_unused uint32_t *table,
9197                                 struct rte_flow_error *error)
9198 {
9199         return rte_flow_error_set(error, ENOTSUP,
9200                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9201                                   "tunnel offload requires DV support");
9202 }
9203
9204 void
9205 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
9206                         __rte_unused  uint16_t port_id)
9207 {
9208 }
9209 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
9210
9211 static void
9212 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
9213 {
9214         int ret;
9215         struct rte_flow_error error;
9216
9217         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9218                 char *item_name;
9219                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
9220                                     sizeof(item_name),
9221                                     (void *)(uintptr_t)item->type, &error);
9222                 if (ret > 0)
9223                         printf("%s ", item_name);
9224                 else
9225                         printf("%d\n", (int)item->type);
9226         }
9227         printf("END\n");
9228 }