drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint64_t node_flags;
 104         /**<
 105          *  Bit-fields that define how the node is used in the expansion.
 106          * (see MLX5_EXPANSION_NODE_* definitions).
 107          */
 108 };
 109
 110 /* Optional expand field. The expansion alg will not go deeper. */
 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
 112
 113 /* The node is not added implicitly as expansion to the flow pattern.
 114  * If the node type does not match the flow pattern item type, the
 115  * expansion alg will go deeper to its next items.
 116  * In the current implementation, the list of next nodes indexes can
 117  * have up to one node with this flag set and it has to be the last
 118  * node index (before the list terminator).
 119  */
 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
 121
 122 /** Object returned by mlx5_flow_expand_rss(). */
 123 struct mlx5_flow_expand_rss {
 124         uint32_t entries;
 125         /**< Number of entries @p patterns and @p priorities. */
 126         struct {
 127                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 128                 uint32_t priority; /**< Priority offset for each expansion. */
 129         } entry[];
 130 };
 131
 132 static void
 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 134
 135 static bool
 136 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 137 {
 138         switch (item->type) {
 139         case RTE_FLOW_ITEM_TYPE_ETH:
 140         case RTE_FLOW_ITEM_TYPE_VLAN:
 141         case RTE_FLOW_ITEM_TYPE_IPV4:
 142         case RTE_FLOW_ITEM_TYPE_IPV6:
 143         case RTE_FLOW_ITEM_TYPE_UDP:
 144         case RTE_FLOW_ITEM_TYPE_TCP:
 145         case RTE_FLOW_ITEM_TYPE_VXLAN:
 146         case RTE_FLOW_ITEM_TYPE_NVGRE:
 147         case RTE_FLOW_ITEM_TYPE_GRE:
 148         case RTE_FLOW_ITEM_TYPE_GENEVE:
 149         case RTE_FLOW_ITEM_TYPE_MPLS:
 150         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 151         case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 152         case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
 153         case RTE_FLOW_ITEM_TYPE_GTP:
 154                 return true;
 155         default:
 156                 break;
 157         }
 158         return false;
 159 }
 160
 161 static enum rte_flow_item_type
 162 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 163 {
 164         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 165         uint16_t ether_type = 0;
 166         uint16_t ether_type_m;
 167         uint8_t ip_next_proto = 0;
 168         uint8_t ip_next_proto_m;
 169
 170         if (item == NULL || item->spec == NULL)
 171                 return ret;
 172         switch (item->type) {
 173         case RTE_FLOW_ITEM_TYPE_ETH:
 174                 if (item->mask)
 175                         ether_type_m = ((const struct rte_flow_item_eth *)
 176                                                 (item->mask))->type;
 177                 else
 178                         ether_type_m = rte_flow_item_eth_mask.type;
 179                 if (ether_type_m != RTE_BE16(0xFFFF))
 180                         break;
 181                 ether_type = ((const struct rte_flow_item_eth *)
 182                                 (item->spec))->type;
 183                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 184                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 185                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 186                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 187                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 188                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 189                 else
 190                         ret = RTE_FLOW_ITEM_TYPE_END;
 191                 break;
 192         case RTE_FLOW_ITEM_TYPE_VLAN:
 193                 if (item->mask)
 194                         ether_type_m = ((const struct rte_flow_item_vlan *)
 195                                                 (item->mask))->inner_type;
 196                 else
 197                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 198                 if (ether_type_m != RTE_BE16(0xFFFF))
 199                         break;
 200                 ether_type = ((const struct rte_flow_item_vlan *)
 201                                 (item->spec))->inner_type;
 202                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 203                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 204                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 205                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 206                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 207                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 208                 else
 209                         ret = RTE_FLOW_ITEM_TYPE_END;
 210                 break;
 211         case RTE_FLOW_ITEM_TYPE_IPV4:
 212                 if (item->mask)
 213                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 214                                         (item->mask))->hdr.next_proto_id;
 215                 else
 216                         ip_next_proto_m =
 217                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 218                 if (ip_next_proto_m != 0xFF)
 219                         break;
 220                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 221                                 (item->spec))->hdr.next_proto_id;
 222                 if (ip_next_proto == IPPROTO_UDP)
 223                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 224                 else if (ip_next_proto == IPPROTO_TCP)
 225                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 226                 else if (ip_next_proto == IPPROTO_IP)
 227                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 228                 else if (ip_next_proto == IPPROTO_IPV6)
 229                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 230                 else
 231                         ret = RTE_FLOW_ITEM_TYPE_END;
 232                 break;
 233         case RTE_FLOW_ITEM_TYPE_IPV6:
 234                 if (item->mask)
 235                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 236                                                 (item->mask))->hdr.proto;
 237                 else
 238                         ip_next_proto_m =
 239                                 rte_flow_item_ipv6_mask.hdr.proto;
 240                 if (ip_next_proto_m != 0xFF)
 241                         break;
 242                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 243                                 (item->spec))->hdr.proto;
 244                 if (ip_next_proto == IPPROTO_UDP)
 245                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 246                 else if (ip_next_proto == IPPROTO_TCP)
 247                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 248                 else if (ip_next_proto == IPPROTO_IP)
 249                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 250                 else if (ip_next_proto == IPPROTO_IPV6)
 251                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 252                 else
 253                         ret = RTE_FLOW_ITEM_TYPE_END;
 254                 break;
 255         default:
 256                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 257                 break;
 258         }
 259         return ret;
 260 }
 261
 262 static const int *
 263 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
 264                 const int *next_node)
 265 {
 266         const struct mlx5_flow_expand_node *node = NULL;
 267         const int *next = next_node;
 268
 269         while (next && *next) {
 270                 /*
 271                  * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
 272                  * flag set, because they were not found in the flow pattern.
 273                  */
 274                 node = &graph[*next];
 275                 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
 276                         break;
 277                 next = node->next;
 278         }
 279         return next;
 280 }
 281
 282 #define MLX5_RSS_EXP_ELT_N 16
 283
 284 /**
 285  * Expand RSS flows into several possible flows according to the RSS hash
 286  * fields requested and the driver capabilities.
 287  *
 288  * @param[out] buf
 289  *   Buffer to store the result expansion.
 290  * @param[in] size
 291  *   Buffer size in bytes. If 0, @p buf can be NULL.
 292  * @param[in] pattern
 293  *   User flow pattern.
 294  * @param[in] types
 295  *   RSS types to expand (see ETH_RSS_* definitions).
 296  * @param[in] graph
 297  *   Input graph to expand @p pattern according to @p types.
 298  * @param[in] graph_root_index
 299  *   Index of root node in @p graph, typically 0.
 300  *
 301  * @return
 302  *   A positive value representing the size of @p buf in bytes regardless of
 303  *   @p size on success, a negative errno value otherwise and rte_errno is
 304  *   set, the following errors are defined:
 305  *
 306  *   -E2BIG: graph-depth @p graph is too deep.
 307  *   -EINVAL: @p size has not enough space for expanded pattern.
 308  */
 309 static int
 310 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 311                      const struct rte_flow_item *pattern, uint64_t types,
 312                      const struct mlx5_flow_expand_node graph[],
 313                      int graph_root_index)
 314 {
 315         const struct rte_flow_item *item;
 316         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 317         const int *next_node;
 318         const int *stack[MLX5_RSS_EXP_ELT_N];
 319         int stack_pos = 0;
 320         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 321         unsigned int i;
 322         size_t lsize;
 323         size_t user_pattern_size = 0;
 324         void *addr = NULL;
 325         const struct mlx5_flow_expand_node *next = NULL;
 326         struct rte_flow_item missed_item;
 327         int missed = 0;
 328         int elt = 0;
 329         const struct rte_flow_item *last_item = NULL;
 330
 331         memset(&missed_item, 0, sizeof(missed_item));
 332         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 333                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 334         if (lsize > size)
 335                 return -EINVAL;
 336         buf->entry[0].priority = 0;
 337         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 338         buf->entries = 0;
 339         addr = buf->entry[0].pattern;
 340         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 341                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 342                         user_pattern_size += sizeof(*item);
 343                         continue;
 344                 }
 345                 last_item = item;
 346                 i = 0;
 347                 while (node->next && node->next[i]) {
 348                         next = &graph[node->next[i]];
 349                         if (next->type == item->type)
 350                                 break;
 351                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 352                                 node = next;
 353                                 i = 0;
 354                         } else {
 355                                 ++i;
 356                         }
 357                 }
 358                 if (next)
 359                         node = next;
 360                 user_pattern_size += sizeof(*item);
 361         }
 362         user_pattern_size += sizeof(*item); /* Handle END item. */
 363         lsize += user_pattern_size;
 364         if (lsize > size)
 365                 return -EINVAL;
 366         /* Copy the user pattern in the first entry of the buffer. */
 367         rte_memcpy(addr, pattern, user_pattern_size);
 368         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 369         buf->entries = 1;
 370         /* Start expanding. */
 371         memset(flow_items, 0, sizeof(flow_items));
 372         user_pattern_size -= sizeof(*item);
 373         /*
 374          * Check if the last valid item has spec set, need complete pattern,
 375          * and the pattern can be used for expansion.
 376          */
 377         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 378         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 379                 /* Item type END indicates expansion is not required. */
 380                 return lsize;
 381         }
 382         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 383                 next = NULL;
 384                 missed = 1;
 385                 for (i = 0; node->next && node->next[i]; ++i) {
 386                         next = &graph[node->next[i]];
 387                         if (next->type == missed_item.type) {
 388                                 flow_items[0].type = missed_item.type;
 389                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 390                                 break;
 391                         }
 392                         next = NULL;
 393                 }
 394         }
 395         if (next && missed) {
 396                 elt = 2; /* missed item + item end. */
 397                 node = next;
 398                 lsize += elt * sizeof(*item) + user_pattern_size;
 399                 if (lsize > size)
 400                         return -EINVAL;
 401                 if (node->rss_types & types) {
 402                         buf->entry[buf->entries].priority = 1;
 403                         buf->entry[buf->entries].pattern = addr;
 404                         buf->entries++;
 405                         rte_memcpy(addr, buf->entry[0].pattern,
 406                                    user_pattern_size);
 407                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 408                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 409                         addr = (void *)(((uintptr_t)addr) +
 410                                         elt * sizeof(*item));
 411                 }
 412         }
 413         memset(flow_items, 0, sizeof(flow_items));
 414         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 415                         node->next);
 416         stack[stack_pos] = next_node;
 417         node = next_node ? &graph[*next_node] : NULL;
 418         while (node) {
 419                 flow_items[stack_pos].type = node->type;
 420                 if (node->rss_types & types) {
 421                         size_t n;
 422                         /*
 423                          * compute the number of items to copy from the
 424                          * expansion and copy it.
 425                          * When the stack_pos is 0, there are 1 element in it,
 426                          * plus the addition END item.
 427                          */
 428                         elt = stack_pos + 2;
 429                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 430                         lsize += elt * sizeof(*item) + user_pattern_size;
 431                         if (lsize > size)
 432                                 return -EINVAL;
 433                         n = elt * sizeof(*item);
 434                         buf->entry[buf->entries].priority =
 435                                 stack_pos + 1 + missed;
 436                         buf->entry[buf->entries].pattern = addr;
 437                         buf->entries++;
 438                         rte_memcpy(addr, buf->entry[0].pattern,
 439                                    user_pattern_size);
 440                         addr = (void *)(((uintptr_t)addr) +
 441                                         user_pattern_size);
 442                         rte_memcpy(addr, &missed_item,
 443                                    missed * sizeof(*item));
 444                         addr = (void *)(((uintptr_t)addr) +
 445                                 missed * sizeof(*item));
 446                         rte_memcpy(addr, flow_items, n);
 447                         addr = (void *)(((uintptr_t)addr) + n);
 448                 }
 449                 /* Go deeper. */
 450                 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 451                                 node->next) {
 452                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 453                                         node->next);
 454                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 455                                 rte_errno = E2BIG;
 456                                 return -rte_errno;
 457                         }
 458                         stack[stack_pos] = next_node;
 459                 } else if (*(next_node + 1)) {
 460                         /* Follow up with the next possibility. */
 461                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 462                                         ++next_node);
 463                 } else {
 464                         /* Move to the next path. */
 465                         if (stack_pos)
 466                                 next_node = stack[--stack_pos];
 467                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 468                                         ++next_node);
 469                         stack[stack_pos] = next_node;
 470                 }
 471                 node = next_node && *next_node ? &graph[*next_node] : NULL;
 472         };
 473         return lsize;
 474 }
 475
 476 enum mlx5_expansion {
 477         MLX5_EXPANSION_ROOT,
 478         MLX5_EXPANSION_ROOT_OUTER,
 479         MLX5_EXPANSION_OUTER_ETH,
 480         MLX5_EXPANSION_OUTER_VLAN,
 481         MLX5_EXPANSION_OUTER_IPV4,
 482         MLX5_EXPANSION_OUTER_IPV4_UDP,
 483         MLX5_EXPANSION_OUTER_IPV4_TCP,
 484         MLX5_EXPANSION_OUTER_IPV6,
 485         MLX5_EXPANSION_OUTER_IPV6_UDP,
 486         MLX5_EXPANSION_OUTER_IPV6_TCP,
 487         MLX5_EXPANSION_VXLAN,
 488         MLX5_EXPANSION_VXLAN_GPE,
 489         MLX5_EXPANSION_GRE,
 490         MLX5_EXPANSION_NVGRE,
 491         MLX5_EXPANSION_GRE_KEY,
 492         MLX5_EXPANSION_MPLS,
 493         MLX5_EXPANSION_ETH,
 494         MLX5_EXPANSION_VLAN,
 495         MLX5_EXPANSION_IPV4,
 496         MLX5_EXPANSION_IPV4_UDP,
 497         MLX5_EXPANSION_IPV4_TCP,
 498         MLX5_EXPANSION_IPV6,
 499         MLX5_EXPANSION_IPV6_UDP,
 500         MLX5_EXPANSION_IPV6_TCP,
 501         MLX5_EXPANSION_IPV6_FRAG_EXT,
 502         MLX5_EXPANSION_GTP
 503 };
 504
 505 /** Supported expansion of items. */
 506 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 507         [MLX5_EXPANSION_ROOT] = {
 508                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 509                                                   MLX5_EXPANSION_IPV4,
 510                                                   MLX5_EXPANSION_IPV6),
 511                 .type = RTE_FLOW_ITEM_TYPE_END,
 512         },
 513         [MLX5_EXPANSION_ROOT_OUTER] = {
 514                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 515                                                   MLX5_EXPANSION_OUTER_IPV4,
 516                                                   MLX5_EXPANSION_OUTER_IPV6),
 517                 .type = RTE_FLOW_ITEM_TYPE_END,
 518         },
 519         [MLX5_EXPANSION_OUTER_ETH] = {
 520                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 521                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 522                 .rss_types = 0,
 523         },
 524         [MLX5_EXPANSION_OUTER_VLAN] = {
 525                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 526                                                   MLX5_EXPANSION_OUTER_IPV6),
 527                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 528                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 529         },
 530         [MLX5_EXPANSION_OUTER_IPV4] = {
 531                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 532                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 533                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 534                          MLX5_EXPANSION_GRE,
 535                          MLX5_EXPANSION_NVGRE,
 536                          MLX5_EXPANSION_IPV4,
 537                          MLX5_EXPANSION_IPV6),
 538                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 539                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 540                         ETH_RSS_NONFRAG_IPV4_OTHER,
 541         },
 542         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 544                                                   MLX5_EXPANSION_VXLAN_GPE,
 545                                                   MLX5_EXPANSION_MPLS,
 546                                                   MLX5_EXPANSION_GTP),
 547                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 548                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 549         },
 550         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 551                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 552                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 553         },
 554         [MLX5_EXPANSION_OUTER_IPV6] = {
 555                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 556                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 557                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 558                          MLX5_EXPANSION_IPV4,
 559                          MLX5_EXPANSION_IPV6,
 560                          MLX5_EXPANSION_GRE,
 561                          MLX5_EXPANSION_NVGRE),
 562                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 563                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 564                         ETH_RSS_NONFRAG_IPV6_OTHER,
 565         },
 566         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 568                                                   MLX5_EXPANSION_VXLAN_GPE,
 569                                                   MLX5_EXPANSION_MPLS,
 570                                                   MLX5_EXPANSION_GTP),
 571                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 572                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 573         },
 574         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 575                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 576                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 577         },
 578         [MLX5_EXPANSION_VXLAN] = {
 579                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 580                                                   MLX5_EXPANSION_IPV4,
 581                                                   MLX5_EXPANSION_IPV6),
 582                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 583         },
 584         [MLX5_EXPANSION_VXLAN_GPE] = {
 585                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 586                                                   MLX5_EXPANSION_IPV4,
 587                                                   MLX5_EXPANSION_IPV6),
 588                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 589         },
 590         [MLX5_EXPANSION_GRE] = {
 591                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 592                                                   MLX5_EXPANSION_IPV6,
 593                                                   MLX5_EXPANSION_GRE_KEY,
 594                                                   MLX5_EXPANSION_MPLS),
 595                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 596         },
 597         [MLX5_EXPANSION_GRE_KEY] = {
 598                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 599                                                   MLX5_EXPANSION_IPV6,
 600                                                   MLX5_EXPANSION_MPLS),
 601                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 602                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 603         },
 604         [MLX5_EXPANSION_NVGRE] = {
 605                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 606                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 607         },
 608         [MLX5_EXPANSION_MPLS] = {
 609                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 610                                                   MLX5_EXPANSION_IPV6,
 611                                                   MLX5_EXPANSION_ETH),
 612                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 613                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 614         },
 615         [MLX5_EXPANSION_ETH] = {
 616                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 617                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 618         },
 619         [MLX5_EXPANSION_VLAN] = {
 620                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 621                                                   MLX5_EXPANSION_IPV6),
 622                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 623                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 624         },
 625         [MLX5_EXPANSION_IPV4] = {
 626                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 627                                                   MLX5_EXPANSION_IPV4_TCP),
 628                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 629                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 630                         ETH_RSS_NONFRAG_IPV4_OTHER,
 631         },
 632         [MLX5_EXPANSION_IPV4_UDP] = {
 633                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 634                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 635         },
 636         [MLX5_EXPANSION_IPV4_TCP] = {
 637                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 638                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 639         },
 640         [MLX5_EXPANSION_IPV6] = {
 641                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 642                                                   MLX5_EXPANSION_IPV6_TCP,
 643                                                   MLX5_EXPANSION_IPV6_FRAG_EXT),
 644                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 645                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 646                         ETH_RSS_NONFRAG_IPV6_OTHER,
 647         },
 648         [MLX5_EXPANSION_IPV6_UDP] = {
 649                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 650                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 651         },
 652         [MLX5_EXPANSION_IPV6_TCP] = {
 653                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 654                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 655         },
 656         [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
 657                 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
 658         },
 659         [MLX5_EXPANSION_GTP] = {
 660                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 661                                                   MLX5_EXPANSION_IPV6),
 662                 .type = RTE_FLOW_ITEM_TYPE_GTP,
 663         },
 664 };
 665
 666 static struct rte_flow_action_handle *
 667 mlx5_action_handle_create(struct rte_eth_dev *dev,
 668                           const struct rte_flow_indir_action_conf *conf,
 669                           const struct rte_flow_action *action,
 670                           struct rte_flow_error *error);
 671 static int mlx5_action_handle_destroy
 672                                 (struct rte_eth_dev *dev,
 673                                  struct rte_flow_action_handle *handle,
 674                                  struct rte_flow_error *error);
 675 static int mlx5_action_handle_update
 676                                 (struct rte_eth_dev *dev,
 677                                  struct rte_flow_action_handle *handle,
 678                                  const void *update,
 679                                  struct rte_flow_error *error);
 680 static int mlx5_action_handle_query
 681                                 (struct rte_eth_dev *dev,
 682                                  const struct rte_flow_action_handle *handle,
 683                                  void *data,
 684                                  struct rte_flow_error *error);
 685 static int
 686 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 687                     struct rte_flow_tunnel *app_tunnel,
 688                     struct rte_flow_action **actions,
 689                     uint32_t *num_of_actions,
 690                     struct rte_flow_error *error);
 691 static int
 692 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 693                        struct rte_flow_tunnel *app_tunnel,
 694                        struct rte_flow_item **items,
 695                        uint32_t *num_of_items,
 696                        struct rte_flow_error *error);
 697 static int
 698 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 699                               struct rte_flow_item *pmd_items,
 700                               uint32_t num_items, struct rte_flow_error *err);
 701 static int
 702 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 703                                 struct rte_flow_action *pmd_actions,
 704                                 uint32_t num_actions,
 705                                 struct rte_flow_error *err);
 706 static int
 707 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 708                                   struct rte_mbuf *m,
 709                                   struct rte_flow_restore_info *info,
 710                                   struct rte_flow_error *err);
 711
 712 static const struct rte_flow_ops mlx5_flow_ops = {
 713         .validate = mlx5_flow_validate,
 714         .create = mlx5_flow_create,
 715         .destroy = mlx5_flow_destroy,
 716         .flush = mlx5_flow_flush,
 717         .isolate = mlx5_flow_isolate,
 718         .query = mlx5_flow_query,
 719         .dev_dump = mlx5_flow_dev_dump,
 720         .get_aged_flows = mlx5_flow_get_aged_flows,
 721         .action_handle_create = mlx5_action_handle_create,
 722         .action_handle_destroy = mlx5_action_handle_destroy,
 723         .action_handle_update = mlx5_action_handle_update,
 724         .action_handle_query = mlx5_action_handle_query,
 725         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 726         .tunnel_match = mlx5_flow_tunnel_match,
 727         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 728         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 729         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 730 };
 731
 732 /* Tunnel information. */
 733 struct mlx5_flow_tunnel_info {
 734         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 735         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 736 };
 737
 738 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 739         {
 740                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 741                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 742         },
 743         {
 744                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 745                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 746         },
 747         {
 748                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 749                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 750         },
 751         {
 752                 .tunnel = MLX5_FLOW_LAYER_GRE,
 753                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 754         },
 755         {
 756                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 757                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 758         },
 759         {
 760                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 761                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 762         },
 763         {
 764                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 765                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 766         },
 767         {
 768                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 769                 .ptype = RTE_PTYPE_TUNNEL_IP,
 770         },
 771         {
 772                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 773                 .ptype = RTE_PTYPE_TUNNEL_IP,
 774         },
 775         {
 776                 .tunnel = MLX5_FLOW_LAYER_GTP,
 777                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 778         },
 779 };
 780
 781
 782
 783 /**
 784  * Translate tag ID to register.
 785  *
 786  * @param[in] dev
 787  *   Pointer to the Ethernet device structure.
 788  * @param[in] feature
 789  *   The feature that request the register.
 790  * @param[in] id
 791  *   The request register ID.
 792  * @param[out] error
 793  *   Error description in case of any.
 794  *
 795  * @return
 796  *   The request register on success, a negative errno
 797  *   value otherwise and rte_errno is set.
 798  */
 799 int
 800 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 801                      enum mlx5_feature_name feature,
 802                      uint32_t id,
 803                      struct rte_flow_error *error)
 804 {
 805         struct mlx5_priv *priv = dev->data->dev_private;
 806         struct mlx5_dev_config *config = &priv->config;
 807         enum modify_reg start_reg;
 808         bool skip_mtr_reg = false;
 809
 810         switch (feature) {
 811         case MLX5_HAIRPIN_RX:
 812                 return REG_B;
 813         case MLX5_HAIRPIN_TX:
 814                 return REG_A;
 815         case MLX5_METADATA_RX:
 816                 switch (config->dv_xmeta_en) {
 817                 case MLX5_XMETA_MODE_LEGACY:
 818                         return REG_B;
 819                 case MLX5_XMETA_MODE_META16:
 820                         return REG_C_0;
 821                 case MLX5_XMETA_MODE_META32:
 822                         return REG_C_1;
 823                 }
 824                 break;
 825         case MLX5_METADATA_TX:
 826                 return REG_A;
 827         case MLX5_METADATA_FDB:
 828                 switch (config->dv_xmeta_en) {
 829                 case MLX5_XMETA_MODE_LEGACY:
 830                         return REG_NON;
 831                 case MLX5_XMETA_MODE_META16:
 832                         return REG_C_0;
 833                 case MLX5_XMETA_MODE_META32:
 834                         return REG_C_1;
 835                 }
 836                 break;
 837         case MLX5_FLOW_MARK:
 838                 switch (config->dv_xmeta_en) {
 839                 case MLX5_XMETA_MODE_LEGACY:
 840                         return REG_NON;
 841                 case MLX5_XMETA_MODE_META16:
 842                         return REG_C_1;
 843                 case MLX5_XMETA_MODE_META32:
 844                         return REG_C_0;
 845                 }
 846                 break;
 847         case MLX5_MTR_ID:
 848                 /*
 849                  * If meter color and meter id share one register, flow match
 850                  * should use the meter color register for match.
 851                  */
 852                 if (priv->mtr_reg_share)
 853                         return priv->mtr_color_reg;
 854                 else
 855                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 856                                REG_C_3;
 857         case MLX5_MTR_COLOR:
 858         case MLX5_ASO_FLOW_HIT:
 859         case MLX5_ASO_CONNTRACK:
 860                 /* All features use the same REG_C. */
 861                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 862                 return priv->mtr_color_reg;
 863         case MLX5_COPY_MARK:
 864                 /*
 865                  * Metadata COPY_MARK register using is in meter suffix sub
 866                  * flow while with meter. It's safe to share the same register.
 867                  */
 868                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 869         case MLX5_APP_TAG:
 870                 /*
 871                  * If meter is enable, it will engage the register for color
 872                  * match and flow match. If meter color match is not using the
 873                  * REG_C_2, need to skip the REG_C_x be used by meter color
 874                  * match.
 875                  * If meter is disable, free to use all available registers.
 876                  */
 877                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 878                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 879                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 880                 if (id > (uint32_t)(REG_C_7 - start_reg))
 881                         return rte_flow_error_set(error, EINVAL,
 882                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 883                                                   NULL, "invalid tag id");
 884                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 885                         return rte_flow_error_set(error, ENOTSUP,
 886                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 887                                                   NULL, "unsupported tag id");
 888                 /*
 889                  * This case means meter is using the REG_C_x great than 2.
 890                  * Take care not to conflict with meter color REG_C_x.
 891                  * If the available index REG_C_y >= REG_C_x, skip the
 892                  * color register.
 893                  */
 894                 if (skip_mtr_reg && config->flow_mreg_c
 895                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 896                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 897                                 return rte_flow_error_set(error, EINVAL,
 898                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 899                                                         NULL, "invalid tag id");
 900                         if (config->flow_mreg_c
 901                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 902                                 return config->flow_mreg_c
 903                                                [id + 1 + start_reg - REG_C_0];
 904                         return rte_flow_error_set(error, ENOTSUP,
 905                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 906                                                   NULL, "unsupported tag id");
 907                 }
 908                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 909         }
 910         MLX5_ASSERT(false);
 911         return rte_flow_error_set(error, EINVAL,
 912                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 913                                   NULL, "invalid feature name");
 914 }
 915
 916 /**
 917  * Check extensive flow metadata register support.
 918  *
 919  * @param dev
 920  *   Pointer to rte_eth_dev structure.
 921  *
 922  * @return
 923  *   True if device supports extensive flow metadata register, otherwise false.
 924  */
 925 bool
 926 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 927 {
 928         struct mlx5_priv *priv = dev->data->dev_private;
 929         struct mlx5_dev_config *config = &priv->config;
 930
 931         /*
 932          * Having available reg_c can be regarded inclusively as supporting
 933          * extensive flow metadata register, which could mean,
 934          * - metadata register copy action by modify header.
 935          * - 16 modify header actions is supported.
 936          * - reg_c's are preserved across different domain (FDB and NIC) on
 937          *   packet loopback by flow lookup miss.
 938          */
 939         return config->flow_mreg_c[2] != REG_NON;
 940 }
 941
 942 /**
 943  * Get the lowest priority.
 944  *
 945  * @param[in] dev
 946  *   Pointer to the Ethernet device structure.
 947  * @param[in] attributes
 948  *   Pointer to device flow rule attributes.
 949  *
 950  * @return
 951  *   The value of lowest priority of flow.
 952  */
 953 uint32_t
 954 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 955                           const struct rte_flow_attr *attr)
 956 {
 957         struct mlx5_priv *priv = dev->data->dev_private;
 958
 959         if (!attr->group && !attr->transfer)
 960                 return priv->config.flow_prio - 2;
 961         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 962 }
 963
 964 /**
 965  * Calculate matcher priority of the flow.
 966  *
 967  * @param[in] dev
 968  *   Pointer to the Ethernet device structure.
 969  * @param[in] attr
 970  *   Pointer to device flow rule attributes.
 971  * @param[in] subpriority
 972  *   The priority based on the items.
 973  * @return
 974  *   The matcher priority of the flow.
 975  */
 976 uint16_t
 977 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 978                           const struct rte_flow_attr *attr,
 979                           uint32_t subpriority)
 980 {
 981         uint16_t priority = (uint16_t)attr->priority;
 982         struct mlx5_priv *priv = dev->data->dev_private;
 983
 984         if (!attr->group && !attr->transfer) {
 985                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 986                         priority = priv->config.flow_prio - 1;
 987                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 988         }
 989         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 990                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 991         return priority * 3 + subpriority;
 992 }
 993
 994 /**
 995  * Verify the @p item specifications (spec, last, mask) are compatible with the
 996  * NIC capabilities.
 997  *
 998  * @param[in] item
 999  *   Item specification.
1000  * @param[in] mask
1001  *   @p item->mask or flow default bit-masks.
1002  * @param[in] nic_mask
1003  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1004  * @param[in] size
1005  *   Bit-masks size in bytes.
1006  * @param[in] range_accepted
1007  *   True if range of values is accepted for specific fields, false otherwise.
1008  * @param[out] error
1009  *   Pointer to error structure.
1010  *
1011  * @return
1012  *   0 on success, a negative errno value otherwise and rte_errno is set.
1013  */
1014 int
1015 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1016                           const uint8_t *mask,
1017                           const uint8_t *nic_mask,
1018                           unsigned int size,
1019                           bool range_accepted,
1020                           struct rte_flow_error *error)
1021 {
1022         unsigned int i;
1023
1024         MLX5_ASSERT(nic_mask);
1025         for (i = 0; i < size; ++i)
1026                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1027                         return rte_flow_error_set(error, ENOTSUP,
1028                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1029                                                   item,
1030                                                   "mask enables non supported"
1031                                                   " bits");
1032         if (!item->spec && (item->mask || item->last))
1033                 return rte_flow_error_set(error, EINVAL,
1034                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1035                                           "mask/last without a spec is not"
1036                                           " supported");
1037         if (item->spec && item->last && !range_accepted) {
1038                 uint8_t spec[size];
1039                 uint8_t last[size];
1040                 unsigned int i;
1041                 int ret;
1042
1043                 for (i = 0; i < size; ++i) {
1044                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1045                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1046                 }
1047                 ret = memcmp(spec, last, size);
1048                 if (ret != 0)
1049                         return rte_flow_error_set(error, EINVAL,
1050                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1051                                                   item,
1052                                                   "range is not valid");
1053         }
1054         return 0;
1055 }
1056
1057 /**
1058  * Adjust the hash fields according to the @p flow information.
1059  *
1060  * @param[in] dev_flow.
1061  *   Pointer to the mlx5_flow.
1062  * @param[in] tunnel
1063  *   1 when the hash field is for a tunnel item.
1064  * @param[in] layer_types
1065  *   ETH_RSS_* types.
1066  * @param[in] hash_fields
1067  *   Item hash fields.
1068  *
1069  * @return
1070  *   The hash fields that should be used.
1071  */
1072 uint64_t
1073 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1074                             int tunnel __rte_unused, uint64_t layer_types,
1075                             uint64_t hash_fields)
1076 {
1077 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1078         int rss_request_inner = rss_desc->level >= 2;
1079
1080         /* Check RSS hash level for tunnel. */
1081         if (tunnel && rss_request_inner)
1082                 hash_fields |= IBV_RX_HASH_INNER;
1083         else if (tunnel || rss_request_inner)
1084                 return 0;
1085 #endif
1086         /* Check if requested layer matches RSS hash fields. */
1087         if (!(rss_desc->types & layer_types))
1088                 return 0;
1089         return hash_fields;
1090 }
1091
1092 /**
1093  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1094  * if several tunnel rules are used on this queue, the tunnel ptype will be
1095  * cleared.
1096  *
1097  * @param rxq_ctrl
1098  *   Rx queue to update.
1099  */
1100 static void
1101 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1102 {
1103         unsigned int i;
1104         uint32_t tunnel_ptype = 0;
1105
1106         /* Look up for the ptype to use. */
1107         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1108                 if (!rxq_ctrl->flow_tunnels_n[i])
1109                         continue;
1110                 if (!tunnel_ptype) {
1111                         tunnel_ptype = tunnels_info[i].ptype;
1112                 } else {
1113                         tunnel_ptype = 0;
1114                         break;
1115                 }
1116         }
1117         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1118 }
1119
1120 /**
1121  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1122  * flow.
1123  *
1124  * @param[in] dev
1125  *   Pointer to the Ethernet device structure.
1126  * @param[in] dev_handle
1127  *   Pointer to device flow handle structure.
1128  */
1129 void
1130 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1131                        struct mlx5_flow_handle *dev_handle)
1132 {
1133         struct mlx5_priv *priv = dev->data->dev_private;
1134         const int mark = dev_handle->mark;
1135         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1136         struct mlx5_ind_table_obj *ind_tbl = NULL;
1137         unsigned int i;
1138
1139         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1140                 struct mlx5_hrxq *hrxq;
1141
1142                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1143                               dev_handle->rix_hrxq);
1144                 if (hrxq)
1145                         ind_tbl = hrxq->ind_table;
1146         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1147                 struct mlx5_shared_action_rss *shared_rss;
1148
1149                 shared_rss = mlx5_ipool_get
1150                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1151                          dev_handle->rix_srss);
1152                 if (shared_rss)
1153                         ind_tbl = shared_rss->ind_tbl;
1154         }
1155         if (!ind_tbl)
1156                 return;
1157         for (i = 0; i != ind_tbl->queues_n; ++i) {
1158                 int idx = ind_tbl->queues[i];
1159                 struct mlx5_rxq_ctrl *rxq_ctrl =
1160                         container_of((*priv->rxqs)[idx],
1161                                      struct mlx5_rxq_ctrl, rxq);
1162
1163                 /*
1164                  * To support metadata register copy on Tx loopback,
1165                  * this must be always enabled (metadata may arive
1166                  * from other port - not from local flows only.
1167                  */
1168                 if (priv->config.dv_flow_en &&
1169                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1170                     mlx5_flow_ext_mreg_supported(dev)) {
1171                         rxq_ctrl->rxq.mark = 1;
1172                         rxq_ctrl->flow_mark_n = 1;
1173                 } else if (mark) {
1174                         rxq_ctrl->rxq.mark = 1;
1175                         rxq_ctrl->flow_mark_n++;
1176                 }
1177                 if (tunnel) {
1178                         unsigned int j;
1179
1180                         /* Increase the counter matching the flow. */
1181                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1182                                 if ((tunnels_info[j].tunnel &
1183                                      dev_handle->layers) ==
1184                                     tunnels_info[j].tunnel) {
1185                                         rxq_ctrl->flow_tunnels_n[j]++;
1186                                         break;
1187                                 }
1188                         }
1189                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1190                 }
1191         }
1192 }
1193
1194 /**
1195  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1196  *
1197  * @param[in] dev
1198  *   Pointer to the Ethernet device structure.
1199  * @param[in] flow
1200  *   Pointer to flow structure.
1201  */
1202 static void
1203 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1204 {
1205         struct mlx5_priv *priv = dev->data->dev_private;
1206         uint32_t handle_idx;
1207         struct mlx5_flow_handle *dev_handle;
1208
1209         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1210                        handle_idx, dev_handle, next)
1211                 flow_drv_rxq_flags_set(dev, dev_handle);
1212 }
1213
1214 /**
1215  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1216  * device flow if no other flow uses it with the same kind of request.
1217  *
1218  * @param dev
1219  *   Pointer to Ethernet device.
1220  * @param[in] dev_handle
1221  *   Pointer to the device flow handle structure.
1222  */
1223 static void
1224 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1225                         struct mlx5_flow_handle *dev_handle)
1226 {
1227         struct mlx5_priv *priv = dev->data->dev_private;
1228         const int mark = dev_handle->mark;
1229         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1230         struct mlx5_ind_table_obj *ind_tbl = NULL;
1231         unsigned int i;
1232
1233         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1234                 struct mlx5_hrxq *hrxq;
1235
1236                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1237                               dev_handle->rix_hrxq);
1238                 if (hrxq)
1239                         ind_tbl = hrxq->ind_table;
1240         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1241                 struct mlx5_shared_action_rss *shared_rss;
1242
1243                 shared_rss = mlx5_ipool_get
1244                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1245                          dev_handle->rix_srss);
1246                 if (shared_rss)
1247                         ind_tbl = shared_rss->ind_tbl;
1248         }
1249         if (!ind_tbl)
1250                 return;
1251         MLX5_ASSERT(dev->data->dev_started);
1252         for (i = 0; i != ind_tbl->queues_n; ++i) {
1253                 int idx = ind_tbl->queues[i];
1254                 struct mlx5_rxq_ctrl *rxq_ctrl =
1255                         container_of((*priv->rxqs)[idx],
1256                                      struct mlx5_rxq_ctrl, rxq);
1257
1258                 if (priv->config.dv_flow_en &&
1259                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1260                     mlx5_flow_ext_mreg_supported(dev)) {
1261                         rxq_ctrl->rxq.mark = 1;
1262                         rxq_ctrl->flow_mark_n = 1;
1263                 } else if (mark) {
1264                         rxq_ctrl->flow_mark_n--;
1265                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1266                 }
1267                 if (tunnel) {
1268                         unsigned int j;
1269
1270                         /* Decrease the counter matching the flow. */
1271                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1272                                 if ((tunnels_info[j].tunnel &
1273                                      dev_handle->layers) ==
1274                                     tunnels_info[j].tunnel) {
1275                                         rxq_ctrl->flow_tunnels_n[j]--;
1276                                         break;
1277                                 }
1278                         }
1279                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1280                 }
1281         }
1282 }
1283
1284 /**
1285  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1286  * @p flow if no other flow uses it with the same kind of request.
1287  *
1288  * @param dev
1289  *   Pointer to Ethernet device.
1290  * @param[in] flow
1291  *   Pointer to the flow.
1292  */
1293 static void
1294 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1295 {
1296         struct mlx5_priv *priv = dev->data->dev_private;
1297         uint32_t handle_idx;
1298         struct mlx5_flow_handle *dev_handle;
1299
1300         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1301                        handle_idx, dev_handle, next)
1302                 flow_drv_rxq_flags_trim(dev, dev_handle);
1303 }
1304
1305 /**
1306  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1307  *
1308  * @param dev
1309  *   Pointer to Ethernet device.
1310  */
1311 static void
1312 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1313 {
1314         struct mlx5_priv *priv = dev->data->dev_private;
1315         unsigned int i;
1316
1317         for (i = 0; i != priv->rxqs_n; ++i) {
1318                 struct mlx5_rxq_ctrl *rxq_ctrl;
1319                 unsigned int j;
1320
1321                 if (!(*priv->rxqs)[i])
1322                         continue;
1323                 rxq_ctrl = container_of((*priv->rxqs)[i],
1324                                         struct mlx5_rxq_ctrl, rxq);
1325                 rxq_ctrl->flow_mark_n = 0;
1326                 rxq_ctrl->rxq.mark = 0;
1327                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1328                         rxq_ctrl->flow_tunnels_n[j] = 0;
1329                 rxq_ctrl->rxq.tunnel = 0;
1330         }
1331 }
1332
1333 /**
1334  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1335  *
1336  * @param[in] dev
1337  *   Pointer to the Ethernet device structure.
1338  */
1339 void
1340 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1341 {
1342         struct mlx5_priv *priv = dev->data->dev_private;
1343         struct mlx5_rxq_data *data;
1344         unsigned int i;
1345
1346         for (i = 0; i != priv->rxqs_n; ++i) {
1347                 if (!(*priv->rxqs)[i])
1348                         continue;
1349                 data = (*priv->rxqs)[i];
1350                 if (!rte_flow_dynf_metadata_avail()) {
1351                         data->dynf_meta = 0;
1352                         data->flow_meta_mask = 0;
1353                         data->flow_meta_offset = -1;
1354                         data->flow_meta_port_mask = 0;
1355                 } else {
1356                         data->dynf_meta = 1;
1357                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1358                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1359                         data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1360                 }
1361         }
1362 }
1363
1364 /*
1365  * return a pointer to the desired action in the list of actions.
1366  *
1367  * @param[in] actions
1368  *   The list of actions to search the action in.
1369  * @param[in] action
1370  *   The action to find.
1371  *
1372  * @return
1373  *   Pointer to the action in the list, if found. NULL otherwise.
1374  */
1375 const struct rte_flow_action *
1376 mlx5_flow_find_action(const struct rte_flow_action *actions,
1377                       enum rte_flow_action_type action)
1378 {
1379         if (actions == NULL)
1380                 return NULL;
1381         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1382                 if (actions->type == action)
1383                         return actions;
1384         return NULL;
1385 }
1386
1387 /*
1388  * Validate the flag action.
1389  *
1390  * @param[in] action_flags
1391  *   Bit-fields that holds the actions detected until now.
1392  * @param[in] attr
1393  *   Attributes of flow that includes this action.
1394  * @param[out] error
1395  *   Pointer to error structure.
1396  *
1397  * @return
1398  *   0 on success, a negative errno value otherwise and rte_errno is set.
1399  */
1400 int
1401 mlx5_flow_validate_action_flag(uint64_t action_flags,
1402                                const struct rte_flow_attr *attr,
1403                                struct rte_flow_error *error)
1404 {
1405         if (action_flags & MLX5_FLOW_ACTION_MARK)
1406                 return rte_flow_error_set(error, EINVAL,
1407                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1408                                           "can't mark and flag in same flow");
1409         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1412                                           "can't have 2 flag"
1413                                           " actions in same flow");
1414         if (attr->egress)
1415                 return rte_flow_error_set(error, ENOTSUP,
1416                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1417                                           "flag action not supported for "
1418                                           "egress");
1419         return 0;
1420 }
1421
1422 /*
1423  * Validate the mark action.
1424  *
1425  * @param[in] action
1426  *   Pointer to the queue action.
1427  * @param[in] action_flags
1428  *   Bit-fields that holds the actions detected until now.
1429  * @param[in] attr
1430  *   Attributes of flow that includes this action.
1431  * @param[out] error
1432  *   Pointer to error structure.
1433  *
1434  * @return
1435  *   0 on success, a negative errno value otherwise and rte_errno is set.
1436  */
1437 int
1438 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1439                                uint64_t action_flags,
1440                                const struct rte_flow_attr *attr,
1441                                struct rte_flow_error *error)
1442 {
1443         const struct rte_flow_action_mark *mark = action->conf;
1444
1445         if (!mark)
1446                 return rte_flow_error_set(error, EINVAL,
1447                                           RTE_FLOW_ERROR_TYPE_ACTION,
1448                                           action,
1449                                           "configuration cannot be null");
1450         if (mark->id >= MLX5_FLOW_MARK_MAX)
1451                 return rte_flow_error_set(error, EINVAL,
1452                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1453                                           &mark->id,
1454                                           "mark id must in 0 <= id < "
1455                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1456         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1457                 return rte_flow_error_set(error, EINVAL,
1458                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1459                                           "can't flag and mark in same flow");
1460         if (action_flags & MLX5_FLOW_ACTION_MARK)
1461                 return rte_flow_error_set(error, EINVAL,
1462                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1463                                           "can't have 2 mark actions in same"
1464                                           " flow");
1465         if (attr->egress)
1466                 return rte_flow_error_set(error, ENOTSUP,
1467                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1468                                           "mark action not supported for "
1469                                           "egress");
1470         return 0;
1471 }
1472
1473 /*
1474  * Validate the drop action.
1475  *
1476  * @param[in] action_flags
1477  *   Bit-fields that holds the actions detected until now.
1478  * @param[in] attr
1479  *   Attributes of flow that includes this action.
1480  * @param[out] error
1481  *   Pointer to error structure.
1482  *
1483  * @return
1484  *   0 on success, a negative errno value otherwise and rte_errno is set.
1485  */
1486 int
1487 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1488                                const struct rte_flow_attr *attr,
1489                                struct rte_flow_error *error)
1490 {
1491         if (attr->egress)
1492                 return rte_flow_error_set(error, ENOTSUP,
1493                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1494                                           "drop action not supported for "
1495                                           "egress");
1496         return 0;
1497 }
1498
1499 /*
1500  * Validate the queue action.
1501  *
1502  * @param[in] action
1503  *   Pointer to the queue action.
1504  * @param[in] action_flags
1505  *   Bit-fields that holds the actions detected until now.
1506  * @param[in] dev
1507  *   Pointer to the Ethernet device structure.
1508  * @param[in] attr
1509  *   Attributes of flow that includes this action.
1510  * @param[out] error
1511  *   Pointer to error structure.
1512  *
1513  * @return
1514  *   0 on success, a negative errno value otherwise and rte_errno is set.
1515  */
1516 int
1517 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1518                                 uint64_t action_flags,
1519                                 struct rte_eth_dev *dev,
1520                                 const struct rte_flow_attr *attr,
1521                                 struct rte_flow_error *error)
1522 {
1523         struct mlx5_priv *priv = dev->data->dev_private;
1524         const struct rte_flow_action_queue *queue = action->conf;
1525
1526         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1527                 return rte_flow_error_set(error, EINVAL,
1528                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1529                                           "can't have 2 fate actions in"
1530                                           " same flow");
1531         if (!priv->rxqs_n)
1532                 return rte_flow_error_set(error, EINVAL,
1533                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1534                                           NULL, "No Rx queues configured");
1535         if (queue->index >= priv->rxqs_n)
1536                 return rte_flow_error_set(error, EINVAL,
1537                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1538                                           &queue->index,
1539                                           "queue index out of range");
1540         if (!(*priv->rxqs)[queue->index])
1541                 return rte_flow_error_set(error, EINVAL,
1542                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1543                                           &queue->index,
1544                                           "queue is not configured");
1545         if (attr->egress)
1546                 return rte_flow_error_set(error, ENOTSUP,
1547                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1548                                           "queue action not supported for "
1549                                           "egress");
1550         return 0;
1551 }
1552
1553 /*
1554  * Validate the rss action.
1555  *
1556  * @param[in] dev
1557  *   Pointer to the Ethernet device structure.
1558  * @param[in] action
1559  *   Pointer to the queue action.
1560  * @param[out] error
1561  *   Pointer to error structure.
1562  *
1563  * @return
1564  *   0 on success, a negative errno value otherwise and rte_errno is set.
1565  */
1566 int
1567 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1568                          const struct rte_flow_action *action,
1569                          struct rte_flow_error *error)
1570 {
1571         struct mlx5_priv *priv = dev->data->dev_private;
1572         const struct rte_flow_action_rss *rss = action->conf;
1573         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1574         unsigned int i;
1575
1576         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1577             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1578                 return rte_flow_error_set(error, ENOTSUP,
1579                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1580                                           &rss->func,
1581                                           "RSS hash function not supported");
1582 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1583         if (rss->level > 2)
1584 #else
1585         if (rss->level > 1)
1586 #endif
1587                 return rte_flow_error_set(error, ENOTSUP,
1588                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1589                                           &rss->level,
1590                                           "tunnel RSS is not supported");
1591         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1592         if (rss->key_len == 0 && rss->key != NULL)
1593                 return rte_flow_error_set(error, ENOTSUP,
1594                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1595                                           &rss->key_len,
1596                                           "RSS hash key length 0");
1597         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1598                 return rte_flow_error_set(error, ENOTSUP,
1599                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1600                                           &rss->key_len,
1601                                           "RSS hash key too small");
1602         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1603                 return rte_flow_error_set(error, ENOTSUP,
1604                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1605                                           &rss->key_len,
1606                                           "RSS hash key too large");
1607         if (rss->queue_num > priv->config.ind_table_max_size)
1608                 return rte_flow_error_set(error, ENOTSUP,
1609                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1610                                           &rss->queue_num,
1611                                           "number of queues too large");
1612         if (rss->types & MLX5_RSS_HF_MASK)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1615                                           &rss->types,
1616                                           "some RSS protocols are not"
1617                                           " supported");
1618         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1619             !(rss->types & ETH_RSS_IP))
1620                 return rte_flow_error_set(error, EINVAL,
1621                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1622                                           "L3 partial RSS requested but L3 RSS"
1623                                           " type not specified");
1624         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1625             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1626                 return rte_flow_error_set(error, EINVAL,
1627                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1628                                           "L4 partial RSS requested but L4 RSS"
1629                                           " type not specified");
1630         if (!priv->rxqs_n)
1631                 return rte_flow_error_set(error, EINVAL,
1632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1633                                           NULL, "No Rx queues configured");
1634         if (!rss->queue_num)
1635                 return rte_flow_error_set(error, EINVAL,
1636                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1637                                           NULL, "No queues configured");
1638         for (i = 0; i != rss->queue_num; ++i) {
1639                 struct mlx5_rxq_ctrl *rxq_ctrl;
1640
1641                 if (rss->queue[i] >= priv->rxqs_n)
1642                         return rte_flow_error_set
1643                                 (error, EINVAL,
1644                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1645                                  &rss->queue[i], "queue index out of range");
1646                 if (!(*priv->rxqs)[rss->queue[i]])
1647                         return rte_flow_error_set
1648                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1649                                  &rss->queue[i], "queue is not configured");
1650                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1651                                         struct mlx5_rxq_ctrl, rxq);
1652                 if (i == 0)
1653                         rxq_type = rxq_ctrl->type;
1654                 if (rxq_type != rxq_ctrl->type)
1655                         return rte_flow_error_set
1656                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1657                                  &rss->queue[i],
1658                                  "combining hairpin and regular RSS queues is not supported");
1659         }
1660         return 0;
1661 }
1662
1663 /*
1664  * Validate the rss action.
1665  *
1666  * @param[in] action
1667  *   Pointer to the queue action.
1668  * @param[in] action_flags
1669  *   Bit-fields that holds the actions detected until now.
1670  * @param[in] dev
1671  *   Pointer to the Ethernet device structure.
1672  * @param[in] attr
1673  *   Attributes of flow that includes this action.
1674  * @param[in] item_flags
1675  *   Items that were detected.
1676  * @param[out] error
1677  *   Pointer to error structure.
1678  *
1679  * @return
1680  *   0 on success, a negative errno value otherwise and rte_errno is set.
1681  */
1682 int
1683 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1684                               uint64_t action_flags,
1685                               struct rte_eth_dev *dev,
1686                               const struct rte_flow_attr *attr,
1687                               uint64_t item_flags,
1688                               struct rte_flow_error *error)
1689 {
1690         const struct rte_flow_action_rss *rss = action->conf;
1691         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1692         int ret;
1693
1694         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1695                 return rte_flow_error_set(error, EINVAL,
1696                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1697                                           "can't have 2 fate actions"
1698                                           " in same flow");
1699         ret = mlx5_validate_action_rss(dev, action, error);
1700         if (ret)
1701                 return ret;
1702         if (attr->egress)
1703                 return rte_flow_error_set(error, ENOTSUP,
1704                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1705                                           "rss action not supported for "
1706                                           "egress");
1707         if (rss->level > 1 && !tunnel)
1708                 return rte_flow_error_set(error, EINVAL,
1709                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1710                                           "inner RSS is not supported for "
1711                                           "non-tunnel flows");
1712         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1713             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1714                 return rte_flow_error_set(error, EINVAL,
1715                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1716                                           "RSS on eCPRI is not supported now");
1717         }
1718         if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
1719             !(item_flags &
1720               (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
1721             rss->level > 1)
1722                 return rte_flow_error_set(error, EINVAL,
1723                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1724                                           "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
1725         return 0;
1726 }
1727
1728 /*
1729  * Validate the default miss action.
1730  *
1731  * @param[in] action_flags
1732  *   Bit-fields that holds the actions detected until now.
1733  * @param[out] error
1734  *   Pointer to error structure.
1735  *
1736  * @return
1737  *   0 on success, a negative errno value otherwise and rte_errno is set.
1738  */
1739 int
1740 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1741                                 const struct rte_flow_attr *attr,
1742                                 struct rte_flow_error *error)
1743 {
1744         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1745                 return rte_flow_error_set(error, EINVAL,
1746                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1747                                           "can't have 2 fate actions in"
1748                                           " same flow");
1749         if (attr->egress)
1750                 return rte_flow_error_set(error, ENOTSUP,
1751                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1752                                           "default miss action not supported "
1753                                           "for egress");
1754         if (attr->group)
1755                 return rte_flow_error_set(error, ENOTSUP,
1756                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1757                                           "only group 0 is supported");
1758         if (attr->transfer)
1759                 return rte_flow_error_set(error, ENOTSUP,
1760                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1761                                           NULL, "transfer is not supported");
1762         return 0;
1763 }
1764
1765 /*
1766  * Validate the count action.
1767  *
1768  * @param[in] dev
1769  *   Pointer to the Ethernet device structure.
1770  * @param[in] attr
1771  *   Attributes of flow that includes this action.
1772  * @param[out] error
1773  *   Pointer to error structure.
1774  *
1775  * @return
1776  *   0 on success, a negative errno value otherwise and rte_errno is set.
1777  */
1778 int
1779 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1780                                 const struct rte_flow_attr *attr,
1781                                 struct rte_flow_error *error)
1782 {
1783         if (attr->egress)
1784                 return rte_flow_error_set(error, ENOTSUP,
1785                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1786                                           "count action not supported for "
1787                                           "egress");
1788         return 0;
1789 }
1790
1791 /*
1792  * Validate the ASO CT action.
1793  *
1794  * @param[in] dev
1795  *   Pointer to the Ethernet device structure.
1796  * @param[in] conntrack
1797  *   Pointer to the CT action profile.
1798  * @param[out] error
1799  *   Pointer to error structure.
1800  *
1801  * @return
1802  *   0 on success, a negative errno value otherwise and rte_errno is set.
1803  */
1804 int
1805 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1806                         const struct rte_flow_action_conntrack *conntrack,
1807                         struct rte_flow_error *error)
1808 {
1809         RTE_SET_USED(dev);
1810
1811         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1812                 return rte_flow_error_set(error, EINVAL,
1813                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1814                                           "Invalid CT state");
1815         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1816                 return rte_flow_error_set(error, EINVAL,
1817                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1818                                           "Invalid last TCP packet flag");
1819         return 0;
1820 }
1821
1822 /**
1823  * Verify the @p attributes will be correctly understood by the NIC and store
1824  * them in the @p flow if everything is correct.
1825  *
1826  * @param[in] dev
1827  *   Pointer to the Ethernet device structure.
1828  * @param[in] attributes
1829  *   Pointer to flow attributes
1830  * @param[out] error
1831  *   Pointer to error structure.
1832  *
1833  * @return
1834  *   0 on success, a negative errno value otherwise and rte_errno is set.
1835  */
1836 int
1837 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1838                               const struct rte_flow_attr *attributes,
1839                               struct rte_flow_error *error)
1840 {
1841         struct mlx5_priv *priv = dev->data->dev_private;
1842         uint32_t priority_max = priv->config.flow_prio - 1;
1843
1844         if (attributes->group)
1845                 return rte_flow_error_set(error, ENOTSUP,
1846                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1847                                           NULL, "groups is not supported");
1848         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1849             attributes->priority >= priority_max)
1850                 return rte_flow_error_set(error, ENOTSUP,
1851                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1852                                           NULL, "priority out of range");
1853         if (attributes->egress)
1854                 return rte_flow_error_set(error, ENOTSUP,
1855                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1856                                           "egress is not supported");
1857         if (attributes->transfer && !priv->config.dv_esw_en)
1858                 return rte_flow_error_set(error, ENOTSUP,
1859                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1860                                           NULL, "transfer is not supported");
1861         if (!attributes->ingress)
1862                 return rte_flow_error_set(error, EINVAL,
1863                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1864                                           NULL,
1865                                           "ingress attribute is mandatory");
1866         return 0;
1867 }
1868
1869 /**
1870  * Validate ICMP6 item.
1871  *
1872  * @param[in] item
1873  *   Item specification.
1874  * @param[in] item_flags
1875  *   Bit-fields that holds the items detected until now.
1876  * @param[in] ext_vlan_sup
1877  *   Whether extended VLAN features are supported or not.
1878  * @param[out] error
1879  *   Pointer to error structure.
1880  *
1881  * @return
1882  *   0 on success, a negative errno value otherwise and rte_errno is set.
1883  */
1884 int
1885 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1886                                uint64_t item_flags,
1887                                uint8_t target_protocol,
1888                                struct rte_flow_error *error)
1889 {
1890         const struct rte_flow_item_icmp6 *mask = item->mask;
1891         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1892         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1893                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1894         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1895                                       MLX5_FLOW_LAYER_OUTER_L4;
1896         int ret;
1897
1898         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1899                 return rte_flow_error_set(error, EINVAL,
1900                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1901                                           "protocol filtering not compatible"
1902                                           " with ICMP6 layer");
1903         if (!(item_flags & l3m))
1904                 return rte_flow_error_set(error, EINVAL,
1905                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1906                                           "IPv6 is mandatory to filter on"
1907                                           " ICMP6");
1908         if (item_flags & l4m)
1909                 return rte_flow_error_set(error, EINVAL,
1910                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1911                                           "multiple L4 layers not supported");
1912         if (!mask)
1913                 mask = &rte_flow_item_icmp6_mask;
1914         ret = mlx5_flow_item_acceptable
1915                 (item, (const uint8_t *)mask,
1916                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1917                  sizeof(struct rte_flow_item_icmp6),
1918                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1919         if (ret < 0)
1920                 return ret;
1921         return 0;
1922 }
1923
1924 /**
1925  * Validate ICMP item.
1926  *
1927  * @param[in] item
1928  *   Item specification.
1929  * @param[in] item_flags
1930  *   Bit-fields that holds the items detected until now.
1931  * @param[out] error
1932  *   Pointer to error structure.
1933  *
1934  * @return
1935  *   0 on success, a negative errno value otherwise and rte_errno is set.
1936  */
1937 int
1938 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1939                              uint64_t item_flags,
1940                              uint8_t target_protocol,
1941                              struct rte_flow_error *error)
1942 {
1943         const struct rte_flow_item_icmp *mask = item->mask;
1944         const struct rte_flow_item_icmp nic_mask = {
1945                 .hdr.icmp_type = 0xff,
1946                 .hdr.icmp_code = 0xff,
1947                 .hdr.icmp_ident = RTE_BE16(0xffff),
1948                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1949         };
1950         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1951         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1952                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1953         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1954                                       MLX5_FLOW_LAYER_OUTER_L4;
1955         int ret;
1956
1957         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1958                 return rte_flow_error_set(error, EINVAL,
1959                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1960                                           "protocol filtering not compatible"
1961                                           " with ICMP layer");
1962         if (!(item_flags & l3m))
1963                 return rte_flow_error_set(error, EINVAL,
1964                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1965                                           "IPv4 is mandatory to filter"
1966                                           " on ICMP");
1967         if (item_flags & l4m)
1968                 return rte_flow_error_set(error, EINVAL,
1969                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1970                                           "multiple L4 layers not supported");
1971         if (!mask)
1972                 mask = &nic_mask;
1973         ret = mlx5_flow_item_acceptable
1974                 (item, (const uint8_t *)mask,
1975                  (const uint8_t *)&nic_mask,
1976                  sizeof(struct rte_flow_item_icmp),
1977                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1978         if (ret < 0)
1979                 return ret;
1980         return 0;
1981 }
1982
1983 /**
1984  * Validate Ethernet item.
1985  *
1986  * @param[in] item
1987  *   Item specification.
1988  * @param[in] item_flags
1989  *   Bit-fields that holds the items detected until now.
1990  * @param[out] error
1991  *   Pointer to error structure.
1992  *
1993  * @return
1994  *   0 on success, a negative errno value otherwise and rte_errno is set.
1995  */
1996 int
1997 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1998                             uint64_t item_flags, bool ext_vlan_sup,
1999                             struct rte_flow_error *error)
2000 {
2001         const struct rte_flow_item_eth *mask = item->mask;
2002         const struct rte_flow_item_eth nic_mask = {
2003                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2004                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2005                 .type = RTE_BE16(0xffff),
2006                 .has_vlan = ext_vlan_sup ? 1 : 0,
2007         };
2008         int ret;
2009         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2010         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2011                                        MLX5_FLOW_LAYER_OUTER_L2;
2012
2013         if (item_flags & ethm)
2014                 return rte_flow_error_set(error, ENOTSUP,
2015                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2016                                           "multiple L2 layers not supported");
2017         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2018             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2019                 return rte_flow_error_set(error, EINVAL,
2020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2021                                           "L2 layer should not follow "
2022                                           "L3 layers");
2023         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2024             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2025                 return rte_flow_error_set(error, EINVAL,
2026                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2027                                           "L2 layer should not follow VLAN");
2028         if (item_flags & MLX5_FLOW_LAYER_GTP)
2029                 return rte_flow_error_set(error, EINVAL,
2030                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2031                                           "L2 layer should not follow GTP");
2032         if (!mask)
2033                 mask = &rte_flow_item_eth_mask;
2034         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2035                                         (const uint8_t *)&nic_mask,
2036                                         sizeof(struct rte_flow_item_eth),
2037                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2038         return ret;
2039 }
2040
2041 /**
2042  * Validate VLAN item.
2043  *
2044  * @param[in] item
2045  *   Item specification.
2046  * @param[in] item_flags
2047  *   Bit-fields that holds the items detected until now.
2048  * @param[in] dev
2049  *   Ethernet device flow is being created on.
2050  * @param[out] error
2051  *   Pointer to error structure.
2052  *
2053  * @return
2054  *   0 on success, a negative errno value otherwise and rte_errno is set.
2055  */
2056 int
2057 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2058                              uint64_t item_flags,
2059                              struct rte_eth_dev *dev,
2060                              struct rte_flow_error *error)
2061 {
2062         const struct rte_flow_item_vlan *spec = item->spec;
2063         const struct rte_flow_item_vlan *mask = item->mask;
2064         const struct rte_flow_item_vlan nic_mask = {
2065                 .tci = RTE_BE16(UINT16_MAX),
2066                 .inner_type = RTE_BE16(UINT16_MAX),
2067         };
2068         uint16_t vlan_tag = 0;
2069         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2070         int ret;
2071         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2072                                         MLX5_FLOW_LAYER_INNER_L4) :
2073                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2074                                         MLX5_FLOW_LAYER_OUTER_L4);
2075         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2076                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2077
2078         if (item_flags & vlanm)
2079                 return rte_flow_error_set(error, EINVAL,
2080                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2081                                           "multiple VLAN layers not supported");
2082         else if ((item_flags & l34m) != 0)
2083                 return rte_flow_error_set(error, EINVAL,
2084                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2085                                           "VLAN cannot follow L3/L4 layer");
2086         if (!mask)
2087                 mask = &rte_flow_item_vlan_mask;
2088         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2089                                         (const uint8_t *)&nic_mask,
2090                                         sizeof(struct rte_flow_item_vlan),
2091                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2092         if (ret)
2093                 return ret;
2094         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2095                 struct mlx5_priv *priv = dev->data->dev_private;
2096
2097                 if (priv->vmwa_context) {
2098                         /*
2099                          * Non-NULL context means we have a virtual machine
2100                          * and SR-IOV enabled, we have to create VLAN interface
2101                          * to make hypervisor to setup E-Switch vport
2102                          * context correctly. We avoid creating the multiple
2103                          * VLAN interfaces, so we cannot support VLAN tag mask.
2104                          */
2105                         return rte_flow_error_set(error, EINVAL,
2106                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2107                                                   item,
2108                                                   "VLAN tag mask is not"
2109                                                   " supported in virtual"
2110                                                   " environment");
2111                 }
2112         }
2113         if (spec) {
2114                 vlan_tag = spec->tci;
2115                 vlan_tag &= mask->tci;
2116         }
2117         /*
2118          * From verbs perspective an empty VLAN is equivalent
2119          * to a packet without VLAN layer.
2120          */
2121         if (!vlan_tag)
2122                 return rte_flow_error_set(error, EINVAL,
2123                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2124                                           item->spec,
2125                                           "VLAN cannot be empty");
2126         return 0;
2127 }
2128
2129 /**
2130  * Validate IPV4 item.
2131  *
2132  * @param[in] item
2133  *   Item specification.
2134  * @param[in] item_flags
2135  *   Bit-fields that holds the items detected until now.
2136  * @param[in] last_item
2137  *   Previous validated item in the pattern items.
2138  * @param[in] ether_type
2139  *   Type in the ethernet layer header (including dot1q).
2140  * @param[in] acc_mask
2141  *   Acceptable mask, if NULL default internal default mask
2142  *   will be used to check whether item fields are supported.
2143  * @param[in] range_accepted
2144  *   True if range of values is accepted for specific fields, false otherwise.
2145  * @param[out] error
2146  *   Pointer to error structure.
2147  *
2148  * @return
2149  *   0 on success, a negative errno value otherwise and rte_errno is set.
2150  */
2151 int
2152 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2153                              uint64_t item_flags,
2154                              uint64_t last_item,
2155                              uint16_t ether_type,
2156                              const struct rte_flow_item_ipv4 *acc_mask,
2157                              bool range_accepted,
2158                              struct rte_flow_error *error)
2159 {
2160         const struct rte_flow_item_ipv4 *mask = item->mask;
2161         const struct rte_flow_item_ipv4 *spec = item->spec;
2162         const struct rte_flow_item_ipv4 nic_mask = {
2163                 .hdr = {
2164                         .src_addr = RTE_BE32(0xffffffff),
2165                         .dst_addr = RTE_BE32(0xffffffff),
2166                         .type_of_service = 0xff,
2167                         .next_proto_id = 0xff,
2168                 },
2169         };
2170         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2171         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2172                                       MLX5_FLOW_LAYER_OUTER_L3;
2173         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2174                                       MLX5_FLOW_LAYER_OUTER_L4;
2175         int ret;
2176         uint8_t next_proto = 0xFF;
2177         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2178                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2179                                   MLX5_FLOW_LAYER_INNER_VLAN);
2180
2181         if ((last_item & l2_vlan) && ether_type &&
2182             ether_type != RTE_ETHER_TYPE_IPV4)
2183                 return rte_flow_error_set(error, EINVAL,
2184                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2185                                           "IPv4 cannot follow L2/VLAN layer "
2186                                           "which ether type is not IPv4");
2187         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2188                 if (mask && spec)
2189                         next_proto = mask->hdr.next_proto_id &
2190                                      spec->hdr.next_proto_id;
2191                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2192                         return rte_flow_error_set(error, EINVAL,
2193                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2194                                                   item,
2195                                                   "multiple tunnel "
2196                                                   "not supported");
2197         }
2198         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2199                 return rte_flow_error_set(error, EINVAL,
2200                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2201                                           "wrong tunnel type - IPv6 specified "
2202                                           "but IPv4 item provided");
2203         if (item_flags & l3m)
2204                 return rte_flow_error_set(error, ENOTSUP,
2205                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2206                                           "multiple L3 layers not supported");
2207         else if (item_flags & l4m)
2208                 return rte_flow_error_set(error, EINVAL,
2209                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2210                                           "L3 cannot follow an L4 layer.");
2211         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2212                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2215                                           "L3 cannot follow an NVGRE layer.");
2216         if (!mask)
2217                 mask = &rte_flow_item_ipv4_mask;
2218         else if (mask->hdr.next_proto_id != 0 &&
2219                  mask->hdr.next_proto_id != 0xff)
2220                 return rte_flow_error_set(error, EINVAL,
2221                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2222                                           "partial mask is not supported"
2223                                           " for protocol");
2224         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2225                                         acc_mask ? (const uint8_t *)acc_mask
2226                                                  : (const uint8_t *)&nic_mask,
2227                                         sizeof(struct rte_flow_item_ipv4),
2228                                         range_accepted, error);
2229         if (ret < 0)
2230                 return ret;
2231         return 0;
2232 }
2233
2234 /**
2235  * Validate IPV6 item.
2236  *
2237  * @param[in] item
2238  *   Item specification.
2239  * @param[in] item_flags
2240  *   Bit-fields that holds the items detected until now.
2241  * @param[in] last_item
2242  *   Previous validated item in the pattern items.
2243  * @param[in] ether_type
2244  *   Type in the ethernet layer header (including dot1q).
2245  * @param[in] acc_mask
2246  *   Acceptable mask, if NULL default internal default mask
2247  *   will be used to check whether item fields are supported.
2248  * @param[out] error
2249  *   Pointer to error structure.
2250  *
2251  * @return
2252  *   0 on success, a negative errno value otherwise and rte_errno is set.
2253  */
2254 int
2255 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2256                              uint64_t item_flags,
2257                              uint64_t last_item,
2258                              uint16_t ether_type,
2259                              const struct rte_flow_item_ipv6 *acc_mask,
2260                              struct rte_flow_error *error)
2261 {
2262         const struct rte_flow_item_ipv6 *mask = item->mask;
2263         const struct rte_flow_item_ipv6 *spec = item->spec;
2264         const struct rte_flow_item_ipv6 nic_mask = {
2265                 .hdr = {
2266                         .src_addr =
2267                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2268                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2269                         .dst_addr =
2270                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2271                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2272                         .vtc_flow = RTE_BE32(0xffffffff),
2273                         .proto = 0xff,
2274                 },
2275         };
2276         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2277         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2278                                       MLX5_FLOW_LAYER_OUTER_L3;
2279         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2280                                       MLX5_FLOW_LAYER_OUTER_L4;
2281         int ret;
2282         uint8_t next_proto = 0xFF;
2283         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2284                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2285                                   MLX5_FLOW_LAYER_INNER_VLAN);
2286
2287         if ((last_item & l2_vlan) && ether_type &&
2288             ether_type != RTE_ETHER_TYPE_IPV6)
2289                 return rte_flow_error_set(error, EINVAL,
2290                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2291                                           "IPv6 cannot follow L2/VLAN layer "
2292                                           "which ether type is not IPv6");
2293         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2294                 next_proto = spec->hdr.proto;
2295         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2296                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2297                         return rte_flow_error_set(error, EINVAL,
2298                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2299                                                   item,
2300                                                   "multiple tunnel "
2301                                                   "not supported");
2302         }
2303         if (next_proto == IPPROTO_HOPOPTS  ||
2304             next_proto == IPPROTO_ROUTING  ||
2305             next_proto == IPPROTO_FRAGMENT ||
2306             next_proto == IPPROTO_ESP      ||
2307             next_proto == IPPROTO_AH       ||
2308             next_proto == IPPROTO_DSTOPTS)
2309                 return rte_flow_error_set(error, EINVAL,
2310                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2311                                           "IPv6 proto (next header) should "
2312                                           "not be set as extension header");
2313         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2314                 return rte_flow_error_set(error, EINVAL,
2315                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2316                                           "wrong tunnel type - IPv4 specified "
2317                                           "but IPv6 item provided");
2318         if (item_flags & l3m)
2319                 return rte_flow_error_set(error, ENOTSUP,
2320                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2321                                           "multiple L3 layers not supported");
2322         else if (item_flags & l4m)
2323                 return rte_flow_error_set(error, EINVAL,
2324                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2325                                           "L3 cannot follow an L4 layer.");
2326         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2327                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2328                 return rte_flow_error_set(error, EINVAL,
2329                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2330                                           "L3 cannot follow an NVGRE layer.");
2331         if (!mask)
2332                 mask = &rte_flow_item_ipv6_mask;
2333         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2334                                         acc_mask ? (const uint8_t *)acc_mask
2335                                                  : (const uint8_t *)&nic_mask,
2336                                         sizeof(struct rte_flow_item_ipv6),
2337                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2338         if (ret < 0)
2339                 return ret;
2340         return 0;
2341 }
2342
2343 /**
2344  * Validate UDP item.
2345  *
2346  * @param[in] item
2347  *   Item specification.
2348  * @param[in] item_flags
2349  *   Bit-fields that holds the items detected until now.
2350  * @param[in] target_protocol
2351  *   The next protocol in the previous item.
2352  * @param[in] flow_mask
2353  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2354  * @param[out] error
2355  *   Pointer to error structure.
2356  *
2357  * @return
2358  *   0 on success, a negative errno value otherwise and rte_errno is set.
2359  */
2360 int
2361 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2362                             uint64_t item_flags,
2363                             uint8_t target_protocol,
2364                             struct rte_flow_error *error)
2365 {
2366         const struct rte_flow_item_udp *mask = item->mask;
2367         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2368         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2369                                       MLX5_FLOW_LAYER_OUTER_L3;
2370         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2371                                       MLX5_FLOW_LAYER_OUTER_L4;
2372         int ret;
2373
2374         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2377                                           "protocol filtering not compatible"
2378                                           " with UDP layer");
2379         if (!(item_flags & l3m))
2380                 return rte_flow_error_set(error, EINVAL,
2381                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2382                                           "L3 is mandatory to filter on L4");
2383         if (item_flags & l4m)
2384                 return rte_flow_error_set(error, EINVAL,
2385                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2386                                           "multiple L4 layers not supported");
2387         if (!mask)
2388                 mask = &rte_flow_item_udp_mask;
2389         ret = mlx5_flow_item_acceptable
2390                 (item, (const uint8_t *)mask,
2391                  (const uint8_t *)&rte_flow_item_udp_mask,
2392                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2393                  error);
2394         if (ret < 0)
2395                 return ret;
2396         return 0;
2397 }
2398
2399 /**
2400  * Validate TCP item.
2401  *
2402  * @param[in] item
2403  *   Item specification.
2404  * @param[in] item_flags
2405  *   Bit-fields that holds the items detected until now.
2406  * @param[in] target_protocol
2407  *   The next protocol in the previous item.
2408  * @param[out] error
2409  *   Pointer to error structure.
2410  *
2411  * @return
2412  *   0 on success, a negative errno value otherwise and rte_errno is set.
2413  */
2414 int
2415 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2416                             uint64_t item_flags,
2417                             uint8_t target_protocol,
2418                             const struct rte_flow_item_tcp *flow_mask,
2419                             struct rte_flow_error *error)
2420 {
2421         const struct rte_flow_item_tcp *mask = item->mask;
2422         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2423         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2424                                       MLX5_FLOW_LAYER_OUTER_L3;
2425         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2426                                       MLX5_FLOW_LAYER_OUTER_L4;
2427         int ret;
2428
2429         MLX5_ASSERT(flow_mask);
2430         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2433                                           "protocol filtering not compatible"
2434                                           " with TCP layer");
2435         if (!(item_flags & l3m))
2436                 return rte_flow_error_set(error, EINVAL,
2437                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2438                                           "L3 is mandatory to filter on L4");
2439         if (item_flags & l4m)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2442                                           "multiple L4 layers not supported");
2443         if (!mask)
2444                 mask = &rte_flow_item_tcp_mask;
2445         ret = mlx5_flow_item_acceptable
2446                 (item, (const uint8_t *)mask,
2447                  (const uint8_t *)flow_mask,
2448                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2449                  error);
2450         if (ret < 0)
2451                 return ret;
2452         return 0;
2453 }
2454
2455 /**
2456  * Validate VXLAN item.
2457  *
2458  * @param[in] dev
2459  *   Pointer to the Ethernet device structure.
2460  * @param[in] udp_dport
2461  *   UDP destination port
2462  * @param[in] item
2463  *   Item specification.
2464  * @param[in] item_flags
2465  *   Bit-fields that holds the items detected until now.
2466  * @param[in] attr
2467  *   Flow rule attributes.
2468  * @param[out] error
2469  *   Pointer to error structure.
2470  *
2471  * @return
2472  *   0 on success, a negative errno value otherwise and rte_errno is set.
2473  */
2474 int
2475 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2476                               uint16_t udp_dport,
2477                               const struct rte_flow_item *item,
2478                               uint64_t item_flags,
2479                               const struct rte_flow_attr *attr,
2480                               struct rte_flow_error *error)
2481 {
2482         const struct rte_flow_item_vxlan *spec = item->spec;
2483         const struct rte_flow_item_vxlan *mask = item->mask;
2484         int ret;
2485         struct mlx5_priv *priv = dev->data->dev_private;
2486         union vni {
2487                 uint32_t vlan_id;
2488                 uint8_t vni[4];
2489         } id = { .vlan_id = 0, };
2490         const struct rte_flow_item_vxlan nic_mask = {
2491                 .vni = "\xff\xff\xff",
2492                 .rsvd1 = 0xff,
2493         };
2494         const struct rte_flow_item_vxlan *valid_mask;
2495
2496         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2497                 return rte_flow_error_set(error, ENOTSUP,
2498                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2499                                           "multiple tunnel layers not"
2500                                           " supported");
2501         valid_mask = &rte_flow_item_vxlan_mask;
2502         /*
2503          * Verify only UDPv4 is present as defined in
2504          * https://tools.ietf.org/html/rfc7348
2505          */
2506         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2507                 return rte_flow_error_set(error, EINVAL,
2508                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2509                                           "no outer UDP layer found");
2510         if (!mask)
2511                 mask = &rte_flow_item_vxlan_mask;
2512
2513         if (priv->sh->steering_format_version !=
2514             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2515             !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2516                 /* FDB domain & NIC domain non-zero group */
2517                 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2518                         valid_mask = &nic_mask;
2519                 /* Group zero in NIC domain */
2520                 if (!attr->group && !attr->transfer &&
2521                     priv->sh->tunnel_header_0_1)
2522                         valid_mask = &nic_mask;
2523         }
2524         ret = mlx5_flow_item_acceptable
2525                 (item, (const uint8_t *)mask,
2526                  (const uint8_t *)valid_mask,
2527                  sizeof(struct rte_flow_item_vxlan),
2528                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2529         if (ret < 0)
2530                 return ret;
2531         if (spec) {
2532                 memcpy(&id.vni[1], spec->vni, 3);
2533                 memcpy(&id.vni[1], mask->vni, 3);
2534         }
2535         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2536                 return rte_flow_error_set(error, ENOTSUP,
2537                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2538                                           "VXLAN tunnel must be fully defined");
2539         return 0;
2540 }
2541
2542 /**
2543  * Validate VXLAN_GPE item.
2544  *
2545  * @param[in] item
2546  *   Item specification.
2547  * @param[in] item_flags
2548  *   Bit-fields that holds the items detected until now.
2549  * @param[in] priv
2550  *   Pointer to the private data structure.
2551  * @param[in] target_protocol
2552  *   The next protocol in the previous item.
2553  * @param[out] error
2554  *   Pointer to error structure.
2555  *
2556  * @return
2557  *   0 on success, a negative errno value otherwise and rte_errno is set.
2558  */
2559 int
2560 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2561                                   uint64_t item_flags,
2562                                   struct rte_eth_dev *dev,
2563                                   struct rte_flow_error *error)
2564 {
2565         struct mlx5_priv *priv = dev->data->dev_private;
2566         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2567         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2568         int ret;
2569         union vni {
2570                 uint32_t vlan_id;
2571                 uint8_t vni[4];
2572         } id = { .vlan_id = 0, };
2573
2574         if (!priv->config.l3_vxlan_en)
2575                 return rte_flow_error_set(error, ENOTSUP,
2576                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2577                                           "L3 VXLAN is not enabled by device"
2578                                           " parameter and/or not configured in"
2579                                           " firmware");
2580         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2581                 return rte_flow_error_set(error, ENOTSUP,
2582                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2583                                           "multiple tunnel layers not"
2584                                           " supported");
2585         /*
2586          * Verify only UDPv4 is present as defined in
2587          * https://tools.ietf.org/html/rfc7348
2588          */
2589         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2590                 return rte_flow_error_set(error, EINVAL,
2591                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2592                                           "no outer UDP layer found");
2593         if (!mask)
2594                 mask = &rte_flow_item_vxlan_gpe_mask;
2595         ret = mlx5_flow_item_acceptable
2596                 (item, (const uint8_t *)mask,
2597                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2598                  sizeof(struct rte_flow_item_vxlan_gpe),
2599                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2600         if (ret < 0)
2601                 return ret;
2602         if (spec) {
2603                 if (spec->protocol)
2604                         return rte_flow_error_set(error, ENOTSUP,
2605                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2606                                                   item,
2607                                                   "VxLAN-GPE protocol"
2608                                                   " not supported");
2609                 memcpy(&id.vni[1], spec->vni, 3);
2610                 memcpy(&id.vni[1], mask->vni, 3);
2611         }
2612         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2613                 return rte_flow_error_set(error, ENOTSUP,
2614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2615                                           "VXLAN-GPE tunnel must be fully"
2616                                           " defined");
2617         return 0;
2618 }
2619 /**
2620  * Validate GRE Key item.
2621  *
2622  * @param[in] item
2623  *   Item specification.
2624  * @param[in] item_flags
2625  *   Bit flags to mark detected items.
2626  * @param[in] gre_item
2627  *   Pointer to gre_item
2628  * @param[out] error
2629  *   Pointer to error structure.
2630  *
2631  * @return
2632  *   0 on success, a negative errno value otherwise and rte_errno is set.
2633  */
2634 int
2635 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2636                                 uint64_t item_flags,
2637                                 const struct rte_flow_item *gre_item,
2638                                 struct rte_flow_error *error)
2639 {
2640         const rte_be32_t *mask = item->mask;
2641         int ret = 0;
2642         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2643         const struct rte_flow_item_gre *gre_spec;
2644         const struct rte_flow_item_gre *gre_mask;
2645
2646         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2647                 return rte_flow_error_set(error, ENOTSUP,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2649                                           "Multiple GRE key not support");
2650         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2651                 return rte_flow_error_set(error, ENOTSUP,
2652                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2653                                           "No preceding GRE header");
2654         if (item_flags & MLX5_FLOW_LAYER_INNER)
2655                 return rte_flow_error_set(error, ENOTSUP,
2656                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2657                                           "GRE key following a wrong item");
2658         gre_mask = gre_item->mask;
2659         if (!gre_mask)
2660                 gre_mask = &rte_flow_item_gre_mask;
2661         gre_spec = gre_item->spec;
2662         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2663                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2664                 return rte_flow_error_set(error, EINVAL,
2665                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2666                                           "Key bit must be on");
2667
2668         if (!mask)
2669                 mask = &gre_key_default_mask;
2670         ret = mlx5_flow_item_acceptable
2671                 (item, (const uint8_t *)mask,
2672                  (const uint8_t *)&gre_key_default_mask,
2673                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2674         return ret;
2675 }
2676
2677 /**
2678  * Validate GRE item.
2679  *
2680  * @param[in] item
2681  *   Item specification.
2682  * @param[in] item_flags
2683  *   Bit flags to mark detected items.
2684  * @param[in] target_protocol
2685  *   The next protocol in the previous item.
2686  * @param[out] error
2687  *   Pointer to error structure.
2688  *
2689  * @return
2690  *   0 on success, a negative errno value otherwise and rte_errno is set.
2691  */
2692 int
2693 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2694                             uint64_t item_flags,
2695                             uint8_t target_protocol,
2696                             struct rte_flow_error *error)
2697 {
2698         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2699         const struct rte_flow_item_gre *mask = item->mask;
2700         int ret;
2701         const struct rte_flow_item_gre nic_mask = {
2702                 .c_rsvd0_ver = RTE_BE16(0xB000),
2703                 .protocol = RTE_BE16(UINT16_MAX),
2704         };
2705
2706         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2707                 return rte_flow_error_set(error, EINVAL,
2708                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2709                                           "protocol filtering not compatible"
2710                                           " with this GRE layer");
2711         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2712                 return rte_flow_error_set(error, ENOTSUP,
2713                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2714                                           "multiple tunnel layers not"
2715                                           " supported");
2716         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2717                 return rte_flow_error_set(error, ENOTSUP,
2718                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2719                                           "L3 Layer is missing");
2720         if (!mask)
2721                 mask = &rte_flow_item_gre_mask;
2722         ret = mlx5_flow_item_acceptable
2723                 (item, (const uint8_t *)mask,
2724                  (const uint8_t *)&nic_mask,
2725                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2726                  error);
2727         if (ret < 0)
2728                 return ret;
2729 #ifndef HAVE_MLX5DV_DR
2730 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2731         if (spec && (spec->protocol & mask->protocol))
2732                 return rte_flow_error_set(error, ENOTSUP,
2733                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2734                                           "without MPLS support the"
2735                                           " specification cannot be used for"
2736                                           " filtering");
2737 #endif
2738 #endif
2739         return 0;
2740 }
2741
2742 /**
2743  * Validate Geneve item.
2744  *
2745  * @param[in] item
2746  *   Item specification.
2747  * @param[in] itemFlags
2748  *   Bit-fields that holds the items detected until now.
2749  * @param[in] enPriv
2750  *   Pointer to the private data structure.
2751  * @param[out] error
2752  *   Pointer to error structure.
2753  *
2754  * @return
2755  *   0 on success, a negative errno value otherwise and rte_errno is set.
2756  */
2757
2758 int
2759 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2760                                uint64_t item_flags,
2761                                struct rte_eth_dev *dev,
2762                                struct rte_flow_error *error)
2763 {
2764         struct mlx5_priv *priv = dev->data->dev_private;
2765         const struct rte_flow_item_geneve *spec = item->spec;
2766         const struct rte_flow_item_geneve *mask = item->mask;
2767         int ret;
2768         uint16_t gbhdr;
2769         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2770                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2771         const struct rte_flow_item_geneve nic_mask = {
2772                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2773                 .vni = "\xff\xff\xff",
2774                 .protocol = RTE_BE16(UINT16_MAX),
2775         };
2776
2777         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2778                 return rte_flow_error_set(error, ENOTSUP,
2779                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2780                                           "L3 Geneve is not enabled by device"
2781                                           " parameter and/or not configured in"
2782                                           " firmware");
2783         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2784                 return rte_flow_error_set(error, ENOTSUP,
2785                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2786                                           "multiple tunnel layers not"
2787                                           " supported");
2788         /*
2789          * Verify only UDPv4 is present as defined in
2790          * https://tools.ietf.org/html/rfc7348
2791          */
2792         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2793                 return rte_flow_error_set(error, EINVAL,
2794                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2795                                           "no outer UDP layer found");
2796         if (!mask)
2797                 mask = &rte_flow_item_geneve_mask;
2798         ret = mlx5_flow_item_acceptable
2799                                   (item, (const uint8_t *)mask,
2800                                    (const uint8_t *)&nic_mask,
2801                                    sizeof(struct rte_flow_item_geneve),
2802                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2803         if (ret)
2804                 return ret;
2805         if (spec) {
2806                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2807                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2808                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2809                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2810                         return rte_flow_error_set(error, ENOTSUP,
2811                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2812                                                   item,
2813                                                   "Geneve protocol unsupported"
2814                                                   " fields are being used");
2815                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2816                         return rte_flow_error_set
2817                                         (error, ENOTSUP,
2818                                          RTE_FLOW_ERROR_TYPE_ITEM,
2819                                          item,
2820                                          "Unsupported Geneve options length");
2821         }
2822         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2823                 return rte_flow_error_set
2824                                     (error, ENOTSUP,
2825                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2826                                      "Geneve tunnel must be fully defined");
2827         return 0;
2828 }
2829
2830 /**
2831  * Validate Geneve TLV option item.
2832  *
2833  * @param[in] item
2834  *   Item specification.
2835  * @param[in] last_item
2836  *   Previous validated item in the pattern items.
2837  * @param[in] geneve_item
2838  *   Previous GENEVE item specification.
2839  * @param[in] dev
2840  *   Pointer to the rte_eth_dev structure.
2841  * @param[out] error
2842  *   Pointer to error structure.
2843  *
2844  * @return
2845  *   0 on success, a negative errno value otherwise and rte_errno is set.
2846  */
2847 int
2848 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2849                                    uint64_t last_item,
2850                                    const struct rte_flow_item *geneve_item,
2851                                    struct rte_eth_dev *dev,
2852                                    struct rte_flow_error *error)
2853 {
2854         struct mlx5_priv *priv = dev->data->dev_private;
2855         struct mlx5_dev_ctx_shared *sh = priv->sh;
2856         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2857         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2858         uint8_t data_max_supported =
2859                         hca_attr->max_geneve_tlv_option_data_len * 4;
2860         struct mlx5_dev_config *config = &priv->config;
2861         const struct rte_flow_item_geneve *geneve_spec;
2862         const struct rte_flow_item_geneve *geneve_mask;
2863         const struct rte_flow_item_geneve_opt *spec = item->spec;
2864         const struct rte_flow_item_geneve_opt *mask = item->mask;
2865         unsigned int i;
2866         unsigned int data_len;
2867         uint8_t tlv_option_len;
2868         uint16_t optlen_m, optlen_v;
2869         const struct rte_flow_item_geneve_opt full_mask = {
2870                 .option_class = RTE_BE16(0xffff),
2871                 .option_type = 0xff,
2872                 .option_len = 0x1f,
2873         };
2874
2875         if (!mask)
2876                 mask = &rte_flow_item_geneve_opt_mask;
2877         if (!spec)
2878                 return rte_flow_error_set
2879                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2880                         "Geneve TLV opt class/type/length must be specified");
2881         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2882                 return rte_flow_error_set
2883                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2884                         "Geneve TLV opt length exceeeds the limit (31)");
2885         /* Check if class type and length masks are full. */
2886         if (full_mask.option_class != mask->option_class ||
2887             full_mask.option_type != mask->option_type ||
2888             full_mask.option_len != (mask->option_len & full_mask.option_len))
2889                 return rte_flow_error_set
2890                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2891                         "Geneve TLV opt class/type/length masks must be full");
2892         /* Check if length is supported */
2893         if ((uint32_t)spec->option_len >
2894                         config->hca_attr.max_geneve_tlv_option_data_len)
2895                 return rte_flow_error_set
2896                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2897                         "Geneve TLV opt length not supported");
2898         if (config->hca_attr.max_geneve_tlv_options > 1)
2899                 DRV_LOG(DEBUG,
2900                         "max_geneve_tlv_options supports more than 1 option");
2901         /* Check GENEVE item preceding. */
2902         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2903                 return rte_flow_error_set
2904                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2905                         "Geneve opt item must be preceded with Geneve item");
2906         geneve_spec = geneve_item->spec;
2907         geneve_mask = geneve_item->mask ? geneve_item->mask :
2908                                           &rte_flow_item_geneve_mask;
2909         /* Check if GENEVE TLV option size doesn't exceed option length */
2910         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2911                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2912                 tlv_option_len = spec->option_len & mask->option_len;
2913                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2914                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2915                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2916                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2917                 if ((optlen_v & optlen_m) <= tlv_option_len)
2918                         return rte_flow_error_set
2919                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2920                                  "GENEVE TLV option length exceeds optlen");
2921         }
2922         /* Check if length is 0 or data is 0. */
2923         if (spec->data == NULL || spec->option_len == 0)
2924                 return rte_flow_error_set
2925                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2926                         "Geneve TLV opt with zero data/length not supported");
2927         /* Check not all data & mask are 0. */
2928         data_len = spec->option_len * 4;
2929         if (mask->data == NULL) {
2930                 for (i = 0; i < data_len; i++)
2931                         if (spec->data[i])
2932                                 break;
2933                 if (i == data_len)
2934                         return rte_flow_error_set(error, ENOTSUP,
2935                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2936                                 "Can't match on Geneve option data 0");
2937         } else {
2938                 for (i = 0; i < data_len; i++)
2939                         if (spec->data[i] & mask->data[i])
2940                                 break;
2941                 if (i == data_len)
2942                         return rte_flow_error_set(error, ENOTSUP,
2943                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2944                                 "Can't match on Geneve option data and mask 0");
2945                 /* Check data mask supported. */
2946                 for (i = data_max_supported; i < data_len ; i++)
2947                         if (mask->data[i])
2948                                 return rte_flow_error_set(error, ENOTSUP,
2949                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2950                                         "Data mask is of unsupported size");
2951         }
2952         /* Check GENEVE option is supported in NIC. */
2953         if (!config->hca_attr.geneve_tlv_opt)
2954                 return rte_flow_error_set
2955                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2956                         "Geneve TLV opt not supported");
2957         /* Check if we already have geneve option with different type/class. */
2958         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2959         geneve_opt_resource = sh->geneve_tlv_option_resource;
2960         if (geneve_opt_resource != NULL)
2961                 if (geneve_opt_resource->option_class != spec->option_class ||
2962                     geneve_opt_resource->option_type != spec->option_type ||
2963                     geneve_opt_resource->length != spec->option_len) {
2964                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2965                         return rte_flow_error_set(error, ENOTSUP,
2966                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2967                                 "Only one Geneve TLV option supported");
2968                 }
2969         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2970         return 0;
2971 }
2972
2973 /**
2974  * Validate MPLS item.
2975  *
2976  * @param[in] dev
2977  *   Pointer to the rte_eth_dev structure.
2978  * @param[in] item
2979  *   Item specification.
2980  * @param[in] item_flags
2981  *   Bit-fields that holds the items detected until now.
2982  * @param[in] prev_layer
2983  *   The protocol layer indicated in previous item.
2984  * @param[out] error
2985  *   Pointer to error structure.
2986  *
2987  * @return
2988  *   0 on success, a negative errno value otherwise and rte_errno is set.
2989  */
2990 int
2991 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2992                              const struct rte_flow_item *item __rte_unused,
2993                              uint64_t item_flags __rte_unused,
2994                              uint64_t prev_layer __rte_unused,
2995                              struct rte_flow_error *error)
2996 {
2997 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2998         const struct rte_flow_item_mpls *mask = item->mask;
2999         struct mlx5_priv *priv = dev->data->dev_private;
3000         int ret;
3001
3002         if (!priv->config.mpls_en)
3003                 return rte_flow_error_set(error, ENOTSUP,
3004                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3005                                           "MPLS not supported or"
3006                                           " disabled in firmware"
3007                                           " configuration.");
3008         /* MPLS over UDP, GRE is allowed */
3009         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3010                             MLX5_FLOW_LAYER_GRE |
3011                             MLX5_FLOW_LAYER_GRE_KEY)))
3012                 return rte_flow_error_set(error, EINVAL,
3013                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3014                                           "protocol filtering not compatible"
3015                                           " with MPLS layer");
3016         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3017         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3018             !(item_flags & MLX5_FLOW_LAYER_GRE))
3019                 return rte_flow_error_set(error, ENOTSUP,
3020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3021                                           "multiple tunnel layers not"
3022                                           " supported");
3023         if (!mask)
3024                 mask = &rte_flow_item_mpls_mask;
3025         ret = mlx5_flow_item_acceptable
3026                 (item, (const uint8_t *)mask,
3027                  (const uint8_t *)&rte_flow_item_mpls_mask,
3028                  sizeof(struct rte_flow_item_mpls),
3029                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3030         if (ret < 0)
3031                 return ret;
3032         return 0;
3033 #else
3034         return rte_flow_error_set(error, ENOTSUP,
3035                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
3036                                   "MPLS is not supported by Verbs, please"
3037                                   " update.");
3038 #endif
3039 }
3040
3041 /**
3042  * Validate NVGRE item.
3043  *
3044  * @param[in] item
3045  *   Item specification.
3046  * @param[in] item_flags
3047  *   Bit flags to mark detected items.
3048  * @param[in] target_protocol
3049  *   The next protocol in the previous item.
3050  * @param[out] error
3051  *   Pointer to error structure.
3052  *
3053  * @return
3054  *   0 on success, a negative errno value otherwise and rte_errno is set.
3055  */
3056 int
3057 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3058                               uint64_t item_flags,
3059                               uint8_t target_protocol,
3060                               struct rte_flow_error *error)
3061 {
3062         const struct rte_flow_item_nvgre *mask = item->mask;
3063         int ret;
3064
3065         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3066                 return rte_flow_error_set(error, EINVAL,
3067                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3068                                           "protocol filtering not compatible"
3069                                           " with this GRE layer");
3070         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3071                 return rte_flow_error_set(error, ENOTSUP,
3072                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3073                                           "multiple tunnel layers not"
3074                                           " supported");
3075         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3076                 return rte_flow_error_set(error, ENOTSUP,
3077                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3078                                           "L3 Layer is missing");
3079         if (!mask)
3080                 mask = &rte_flow_item_nvgre_mask;
3081         ret = mlx5_flow_item_acceptable
3082                 (item, (const uint8_t *)mask,
3083                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3084                  sizeof(struct rte_flow_item_nvgre),
3085                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3086         if (ret < 0)
3087                 return ret;
3088         return 0;
3089 }
3090
3091 /**
3092  * Validate eCPRI item.
3093  *
3094  * @param[in] item
3095  *   Item specification.
3096  * @param[in] item_flags
3097  *   Bit-fields that holds the items detected until now.
3098  * @param[in] last_item
3099  *   Previous validated item in the pattern items.
3100  * @param[in] ether_type
3101  *   Type in the ethernet layer header (including dot1q).
3102  * @param[in] acc_mask
3103  *   Acceptable mask, if NULL default internal default mask
3104  *   will be used to check whether item fields are supported.
3105  * @param[out] error
3106  *   Pointer to error structure.
3107  *
3108  * @return
3109  *   0 on success, a negative errno value otherwise and rte_errno is set.
3110  */
3111 int
3112 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3113                               uint64_t item_flags,
3114                               uint64_t last_item,
3115                               uint16_t ether_type,
3116                               const struct rte_flow_item_ecpri *acc_mask,
3117                               struct rte_flow_error *error)
3118 {
3119         const struct rte_flow_item_ecpri *mask = item->mask;
3120         const struct rte_flow_item_ecpri nic_mask = {
3121                 .hdr = {
3122                         .common = {
3123                                 .u32 =
3124                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3125                                         .type = 0xFF,
3126                                         }).u32),
3127                         },
3128                         .dummy[0] = 0xFFFFFFFF,
3129                 },
3130         };
3131         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3132                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3133         struct rte_flow_item_ecpri mask_lo;
3134
3135         if (!(last_item & outer_l2_vlan) &&
3136             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3137                 return rte_flow_error_set(error, EINVAL,
3138                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3139                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3140         if ((last_item & outer_l2_vlan) && ether_type &&
3141             ether_type != RTE_ETHER_TYPE_ECPRI)
3142                 return rte_flow_error_set(error, EINVAL,
3143                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3144                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3145         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3146                 return rte_flow_error_set(error, EINVAL,
3147                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3148                                           "eCPRI with tunnel is not supported right now");
3149         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3150                 return rte_flow_error_set(error, ENOTSUP,
3151                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3152                                           "multiple L3 layers not supported");
3153         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3154                 return rte_flow_error_set(error, EINVAL,
3155                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3156                                           "eCPRI cannot coexist with a TCP layer");
3157         /* In specification, eCPRI could be over UDP layer. */
3158         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3159                 return rte_flow_error_set(error, EINVAL,
3160                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3161                                           "eCPRI over UDP layer is not yet supported right now");
3162         /* Mask for type field in common header could be zero. */
3163         if (!mask)
3164                 mask = &rte_flow_item_ecpri_mask;
3165         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3166         /* Input mask is in big-endian format. */
3167         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3168                 return rte_flow_error_set(error, EINVAL,
3169                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3170                                           "partial mask is not supported for protocol");
3171         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3172                 return rte_flow_error_set(error, EINVAL,
3173                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3174                                           "message header mask must be after a type mask");
3175         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3176                                          acc_mask ? (const uint8_t *)acc_mask
3177                                                   : (const uint8_t *)&nic_mask,
3178                                          sizeof(struct rte_flow_item_ecpri),
3179                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3180 }
3181
3182 static int
3183 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3184                    const struct rte_flow_attr *attr __rte_unused,
3185                    const struct rte_flow_item items[] __rte_unused,
3186                    const struct rte_flow_action actions[] __rte_unused,
3187                    bool external __rte_unused,
3188                    int hairpin __rte_unused,
3189                    struct rte_flow_error *error)
3190 {
3191         return rte_flow_error_set(error, ENOTSUP,
3192                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3193 }
3194
3195 static struct mlx5_flow *
3196 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3197                   const struct rte_flow_attr *attr __rte_unused,
3198                   const struct rte_flow_item items[] __rte_unused,
3199                   const struct rte_flow_action actions[] __rte_unused,
3200                   struct rte_flow_error *error)
3201 {
3202         rte_flow_error_set(error, ENOTSUP,
3203                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3204         return NULL;
3205 }
3206
3207 static int
3208 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3209                     struct mlx5_flow *dev_flow __rte_unused,
3210                     const struct rte_flow_attr *attr __rte_unused,
3211                     const struct rte_flow_item items[] __rte_unused,
3212                     const struct rte_flow_action actions[] __rte_unused,
3213                     struct rte_flow_error *error)
3214 {
3215         return rte_flow_error_set(error, ENOTSUP,
3216                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3217 }
3218
3219 static int
3220 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3221                 struct rte_flow *flow __rte_unused,
3222                 struct rte_flow_error *error)
3223 {
3224         return rte_flow_error_set(error, ENOTSUP,
3225                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3226 }
3227
3228 static void
3229 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3230                  struct rte_flow *flow __rte_unused)
3231 {
3232 }
3233
3234 static void
3235 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3236                   struct rte_flow *flow __rte_unused)
3237 {
3238 }
3239
3240 static int
3241 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3242                 struct rte_flow *flow __rte_unused,
3243                 const struct rte_flow_action *actions __rte_unused,
3244                 void *data __rte_unused,
3245                 struct rte_flow_error *error)
3246 {
3247         return rte_flow_error_set(error, ENOTSUP,
3248                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3249 }
3250
3251 static int
3252 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3253                       uint32_t domains __rte_unused,
3254                       uint32_t flags __rte_unused)
3255 {
3256         return 0;
3257 }
3258
3259 /* Void driver to protect from null pointer reference. */
3260 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3261         .validate = flow_null_validate,
3262         .prepare = flow_null_prepare,
3263         .translate = flow_null_translate,
3264         .apply = flow_null_apply,
3265         .remove = flow_null_remove,
3266         .destroy = flow_null_destroy,
3267         .query = flow_null_query,
3268         .sync_domain = flow_null_sync_domain,
3269 };
3270
3271 /**
3272  * Select flow driver type according to flow attributes and device
3273  * configuration.
3274  *
3275  * @param[in] dev
3276  *   Pointer to the dev structure.
3277  * @param[in] attr
3278  *   Pointer to the flow attributes.
3279  *
3280  * @return
3281  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3282  */
3283 static enum mlx5_flow_drv_type
3284 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3285 {
3286         struct mlx5_priv *priv = dev->data->dev_private;
3287         /* The OS can determine first a specific flow type (DV, VERBS) */
3288         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3289
3290         if (type != MLX5_FLOW_TYPE_MAX)
3291                 return type;
3292         /* If no OS specific type - continue with DV/VERBS selection */
3293         if (attr->transfer && priv->config.dv_esw_en)
3294                 type = MLX5_FLOW_TYPE_DV;
3295         if (!attr->transfer)
3296                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3297                                                  MLX5_FLOW_TYPE_VERBS;
3298         return type;
3299 }
3300
3301 #define flow_get_drv_ops(type) flow_drv_ops[type]
3302
3303 /**
3304  * Flow driver validation API. This abstracts calling driver specific functions.
3305  * The type of flow driver is determined according to flow attributes.
3306  *
3307  * @param[in] dev
3308  *   Pointer to the dev structure.
3309  * @param[in] attr
3310  *   Pointer to the flow attributes.
3311  * @param[in] items
3312  *   Pointer to the list of items.
3313  * @param[in] actions
3314  *   Pointer to the list of actions.
3315  * @param[in] external
3316  *   This flow rule is created by request external to PMD.
3317  * @param[in] hairpin
3318  *   Number of hairpin TX actions, 0 means classic flow.
3319  * @param[out] error
3320  *   Pointer to the error structure.
3321  *
3322  * @return
3323  *   0 on success, a negative errno value otherwise and rte_errno is set.
3324  */
3325 static inline int
3326 flow_drv_validate(struct rte_eth_dev *dev,
3327                   const struct rte_flow_attr *attr,
3328                   const struct rte_flow_item items[],
3329                   const struct rte_flow_action actions[],
3330                   bool external, int hairpin, struct rte_flow_error *error)
3331 {
3332         const struct mlx5_flow_driver_ops *fops;
3333         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3334
3335         fops = flow_get_drv_ops(type);
3336         return fops->validate(dev, attr, items, actions, external,
3337                               hairpin, error);
3338 }
3339
3340 /**
3341  * Flow driver preparation API. This abstracts calling driver specific
3342  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3343  * calculates the size of memory required for device flow, allocates the memory,
3344  * initializes the device flow and returns the pointer.
3345  *
3346  * @note
3347  *   This function initializes device flow structure such as dv or verbs in
3348  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3349  *   rest. For example, adding returning device flow to flow->dev_flow list and
3350  *   setting backward reference to the flow should be done out of this function.
3351  *   layers field is not filled either.
3352  *
3353  * @param[in] dev
3354  *   Pointer to the dev structure.
3355  * @param[in] attr
3356  *   Pointer to the flow attributes.
3357  * @param[in] items
3358  *   Pointer to the list of items.
3359  * @param[in] actions
3360  *   Pointer to the list of actions.
3361  * @param[in] flow_idx
3362  *   This memory pool index to the flow.
3363  * @param[out] error
3364  *   Pointer to the error structure.
3365  *
3366  * @return
3367  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3368  */
3369 static inline struct mlx5_flow *
3370 flow_drv_prepare(struct rte_eth_dev *dev,
3371                  const struct rte_flow *flow,
3372                  const struct rte_flow_attr *attr,
3373                  const struct rte_flow_item items[],
3374                  const struct rte_flow_action actions[],
3375                  uint32_t flow_idx,
3376                  struct rte_flow_error *error)
3377 {
3378         const struct mlx5_flow_driver_ops *fops;
3379         enum mlx5_flow_drv_type type = flow->drv_type;
3380         struct mlx5_flow *mlx5_flow = NULL;
3381
3382         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3383         fops = flow_get_drv_ops(type);
3384         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3385         if (mlx5_flow)
3386                 mlx5_flow->flow_idx = flow_idx;
3387         return mlx5_flow;
3388 }
3389
3390 /**
3391  * Flow driver translation API. This abstracts calling driver specific
3392  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3393  * translates a generic flow into a driver flow. flow_drv_prepare() must
3394  * precede.
3395  *
3396  * @note
3397  *   dev_flow->layers could be filled as a result of parsing during translation
3398  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3399  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3400  *   flow->actions could be overwritten even though all the expanded dev_flows
3401  *   have the same actions.
3402  *
3403  * @param[in] dev
3404  *   Pointer to the rte dev structure.
3405  * @param[in, out] dev_flow
3406  *   Pointer to the mlx5 flow.
3407  * @param[in] attr
3408  *   Pointer to the flow attributes.
3409  * @param[in] items
3410  *   Pointer to the list of items.
3411  * @param[in] actions
3412  *   Pointer to the list of actions.
3413  * @param[out] error
3414  *   Pointer to the error structure.
3415  *
3416  * @return
3417  *   0 on success, a negative errno value otherwise and rte_errno is set.
3418  */
3419 static inline int
3420 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3421                    const struct rte_flow_attr *attr,
3422                    const struct rte_flow_item items[],
3423                    const struct rte_flow_action actions[],
3424                    struct rte_flow_error *error)
3425 {
3426         const struct mlx5_flow_driver_ops *fops;
3427         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3428
3429         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3430         fops = flow_get_drv_ops(type);
3431         return fops->translate(dev, dev_flow, attr, items, actions, error);
3432 }
3433
3434 /**
3435  * Flow driver apply API. This abstracts calling driver specific functions.
3436  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3437  * translated driver flows on to device. flow_drv_translate() must precede.
3438  *
3439  * @param[in] dev
3440  *   Pointer to Ethernet device structure.
3441  * @param[in, out] flow
3442  *   Pointer to flow structure.
3443  * @param[out] error
3444  *   Pointer to error structure.
3445  *
3446  * @return
3447  *   0 on success, a negative errno value otherwise and rte_errno is set.
3448  */
3449 static inline int
3450 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3451                struct rte_flow_error *error)
3452 {
3453         const struct mlx5_flow_driver_ops *fops;
3454         enum mlx5_flow_drv_type type = flow->drv_type;
3455
3456         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3457         fops = flow_get_drv_ops(type);
3458         return fops->apply(dev, flow, error);
3459 }
3460
3461 /**
3462  * Flow driver destroy API. This abstracts calling driver specific functions.
3463  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3464  * on device and releases resources of the flow.
3465  *
3466  * @param[in] dev
3467  *   Pointer to Ethernet device.
3468  * @param[in, out] flow
3469  *   Pointer to flow structure.
3470  */
3471 static inline void
3472 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3473 {
3474         const struct mlx5_flow_driver_ops *fops;
3475         enum mlx5_flow_drv_type type = flow->drv_type;
3476
3477         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3478         fops = flow_get_drv_ops(type);
3479         fops->destroy(dev, flow);
3480 }
3481
3482 /**
3483  * Flow driver find RSS policy tbl API. This abstracts calling driver
3484  * specific functions. Parent flow (rte_flow) should have driver
3485  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3486  *
3487  * @param[in] dev
3488  *   Pointer to Ethernet device.
3489  * @param[in, out] flow
3490  *   Pointer to flow structure.
3491  * @param[in] policy
3492  *   Pointer to meter policy table.
3493  * @param[in] rss_desc
3494  *   Pointer to rss_desc
3495  */
3496 static struct mlx5_flow_meter_sub_policy *
3497 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3498                 struct rte_flow *flow,
3499                 struct mlx5_flow_meter_policy *policy,
3500                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3501 {
3502         const struct mlx5_flow_driver_ops *fops;
3503         enum mlx5_flow_drv_type type = flow->drv_type;
3504
3505         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3506         fops = flow_get_drv_ops(type);
3507         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3508 }
3509
3510 /**
3511  * Flow driver color tag rule API. This abstracts calling driver
3512  * specific functions. Parent flow (rte_flow) should have driver
3513  * type (drv_type). It will create the color tag rules in hierarchy meter.
3514  *
3515  * @param[in] dev
3516  *   Pointer to Ethernet device.
3517  * @param[in, out] flow
3518  *   Pointer to flow structure.
3519  * @param[in] fm
3520  *   Pointer to flow meter structure.
3521  * @param[in] src_port
3522  *   The src port this extra rule should use.
3523  * @param[in] item
3524  *   The src port id match item.
3525  * @param[out] error
3526  *   Pointer to error structure.
3527  */
3528 static int
3529 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3530                 struct rte_flow *flow,
3531                 struct mlx5_flow_meter_info *fm,
3532                 int32_t src_port,
3533                 const struct rte_flow_item *item,
3534                 struct rte_flow_error *error)
3535 {
3536         const struct mlx5_flow_driver_ops *fops;
3537         enum mlx5_flow_drv_type type = flow->drv_type;
3538
3539         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3540         fops = flow_get_drv_ops(type);
3541         return fops->meter_hierarchy_rule_create(dev, fm,
3542                                                 src_port, item, error);
3543 }
3544
3545 /**
3546  * Get RSS action from the action list.
3547  *
3548  * @param[in] dev
3549  *   Pointer to Ethernet device.
3550  * @param[in] actions
3551  *   Pointer to the list of actions.
3552  * @param[in] flow
3553  *   Parent flow structure pointer.
3554  *
3555  * @return
3556  *   Pointer to the RSS action if exist, else return NULL.
3557  */
3558 static const struct rte_flow_action_rss*
3559 flow_get_rss_action(struct rte_eth_dev *dev,
3560                     const struct rte_flow_action actions[])
3561 {
3562         struct mlx5_priv *priv = dev->data->dev_private;
3563         const struct rte_flow_action_rss *rss = NULL;
3564
3565         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3566                 switch (actions->type) {
3567                 case RTE_FLOW_ACTION_TYPE_RSS:
3568                         rss = actions->conf;
3569                         break;
3570                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3571                 {
3572                         const struct rte_flow_action_sample *sample =
3573                                                                 actions->conf;
3574                         const struct rte_flow_action *act = sample->actions;
3575                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3576                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3577                                         rss = act->conf;
3578                         break;
3579                 }
3580                 case RTE_FLOW_ACTION_TYPE_METER:
3581                 {
3582                         uint32_t mtr_idx;
3583                         struct mlx5_flow_meter_info *fm;
3584                         struct mlx5_flow_meter_policy *policy;
3585                         const struct rte_flow_action_meter *mtr = actions->conf;
3586
3587                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3588                         if (fm && !fm->def_policy) {
3589                                 policy = mlx5_flow_meter_policy_find(dev,
3590                                                 fm->policy_id, NULL);
3591                                 MLX5_ASSERT(policy);
3592                                 if (policy->is_hierarchy) {
3593                                         policy =
3594                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
3595                                                                         policy);
3596                                         if (!policy)
3597                                                 return NULL;
3598                                 }
3599                                 if (policy->is_rss)
3600                                         rss =
3601                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3602                         }
3603                         break;
3604                 }
3605                 default:
3606                         break;
3607                 }
3608         }
3609         return rss;
3610 }
3611
3612 /**
3613  * Get ASO age action by index.
3614  *
3615  * @param[in] dev
3616  *   Pointer to the Ethernet device structure.
3617  * @param[in] age_idx
3618  *   Index to the ASO age action.
3619  *
3620  * @return
3621  *   The specified ASO age action.
3622  */
3623 struct mlx5_aso_age_action*
3624 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3625 {
3626         uint16_t pool_idx = age_idx & UINT16_MAX;
3627         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3628         struct mlx5_priv *priv = dev->data->dev_private;
3629         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3630         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3631
3632         return &pool->actions[offset - 1];
3633 }
3634
3635 /* maps indirect action to translated direct in some actions array */
3636 struct mlx5_translated_action_handle {
3637         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3638         int index; /**< Index in related array of rte_flow_action. */
3639 };
3640
3641 /**
3642  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3643  * direct action if translation possible.
3644  * This functionality used to run same execution path for both direct and
3645  * indirect actions on flow create. All necessary preparations for indirect
3646  * action handling should be performed on *handle* actions list returned
3647  * from this call.
3648  *
3649  * @param[in] dev
3650  *   Pointer to Ethernet device.
3651  * @param[in] actions
3652  *   List of actions to translate.
3653  * @param[out] handle
3654  *   List to store translated indirect action object handles.
3655  * @param[in, out] indir_n
3656  *   Size of *handle* array. On return should be updated with number of
3657  *   indirect actions retrieved from the *actions* list.
3658  * @param[out] translated_actions
3659  *   List of actions where all indirect actions were translated to direct
3660  *   if possible. NULL if no translation took place.
3661  * @param[out] error
3662  *   Pointer to the error structure.
3663  *
3664  * @return
3665  *   0 on success, a negative errno value otherwise and rte_errno is set.
3666  */
3667 static int
3668 flow_action_handles_translate(struct rte_eth_dev *dev,
3669                               const struct rte_flow_action actions[],
3670                               struct mlx5_translated_action_handle *handle,
3671                               int *indir_n,
3672                               struct rte_flow_action **translated_actions,
3673                               struct rte_flow_error *error)
3674 {
3675         struct mlx5_priv *priv = dev->data->dev_private;
3676         struct rte_flow_action *translated = NULL;
3677         size_t actions_size;
3678         int n;
3679         int copied_n = 0;
3680         struct mlx5_translated_action_handle *handle_end = NULL;
3681
3682         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3683                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3684                         continue;
3685                 if (copied_n == *indir_n) {
3686                         return rte_flow_error_set
3687                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3688                                  NULL, "too many shared actions");
3689                 }
3690                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3691                            sizeof(actions[n].conf));
3692                 handle[copied_n].index = n;
3693                 copied_n++;
3694         }
3695         n++;
3696         *indir_n = copied_n;
3697         if (!copied_n)
3698                 return 0;
3699         actions_size = sizeof(struct rte_flow_action) * n;
3700         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3701         if (!translated) {
3702                 rte_errno = ENOMEM;
3703                 return -ENOMEM;
3704         }
3705         memcpy(translated, actions, actions_size);
3706         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3707                 struct mlx5_shared_action_rss *shared_rss;
3708                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3709                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3710                 uint32_t idx = act_idx &
3711                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3712
3713                 switch (type) {
3714                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3715                         shared_rss = mlx5_ipool_get
3716                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3717                         translated[handle->index].type =
3718                                 RTE_FLOW_ACTION_TYPE_RSS;
3719                         translated[handle->index].conf =
3720                                 &shared_rss->origin;
3721                         break;
3722                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3723                         translated[handle->index].type =
3724                                                 (enum rte_flow_action_type)
3725                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3726                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3727                         break;
3728                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3729                         if (priv->sh->flow_hit_aso_en) {
3730                                 translated[handle->index].type =
3731                                         (enum rte_flow_action_type)
3732                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3733                                 translated[handle->index].conf =
3734                                                          (void *)(uintptr_t)idx;
3735                                 break;
3736                         }
3737                         /* Fall-through */
3738                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3739                         if (priv->sh->ct_aso_en) {
3740                                 translated[handle->index].type =
3741                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3742                                 translated[handle->index].conf =
3743                                                          (void *)(uintptr_t)idx;
3744                                 break;
3745                         }
3746                         /* Fall-through */
3747                 default:
3748                         mlx5_free(translated);
3749                         return rte_flow_error_set
3750                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3751                                  NULL, "invalid indirect action type");
3752                 }
3753         }
3754         *translated_actions = translated;
3755         return 0;
3756 }
3757
3758 /**
3759  * Get Shared RSS action from the action list.
3760  *
3761  * @param[in] dev
3762  *   Pointer to Ethernet device.
3763  * @param[in] shared
3764  *   Pointer to the list of actions.
3765  * @param[in] shared_n
3766  *   Actions list length.
3767  *
3768  * @return
3769  *   The MLX5 RSS action ID if exists, otherwise return 0.
3770  */
3771 static uint32_t
3772 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3773                            struct mlx5_translated_action_handle *handle,
3774                            int shared_n)
3775 {
3776         struct mlx5_translated_action_handle *handle_end;
3777         struct mlx5_priv *priv = dev->data->dev_private;
3778         struct mlx5_shared_action_rss *shared_rss;
3779
3780
3781         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3782                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3783                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3784                 uint32_t idx = act_idx &
3785                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3786                 switch (type) {
3787                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3788                         shared_rss = mlx5_ipool_get
3789                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3790                                                                            idx);
3791                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3792                                            __ATOMIC_RELAXED);
3793                         return idx;
3794                 default:
3795                         break;
3796                 }
3797         }
3798         return 0;
3799 }
3800
3801 static unsigned int
3802 find_graph_root(uint32_t rss_level)
3803 {
3804         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3805                                MLX5_EXPANSION_ROOT_OUTER;
3806 }
3807
3808 /**
3809  *  Get layer flags from the prefix flow.
3810  *
3811  *  Some flows may be split to several subflows, the prefix subflow gets the
3812  *  match items and the suffix sub flow gets the actions.
3813  *  Some actions need the user defined match item flags to get the detail for
3814  *  the action.
3815  *  This function helps the suffix flow to get the item layer flags from prefix
3816  *  subflow.
3817  *
3818  * @param[in] dev_flow
3819  *   Pointer the created preifx subflow.
3820  *
3821  * @return
3822  *   The layers get from prefix subflow.
3823  */
3824 static inline uint64_t
3825 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3826 {
3827         uint64_t layers = 0;
3828
3829         /*
3830          * Layers bits could be localization, but usually the compiler will
3831          * help to do the optimization work for source code.
3832          * If no decap actions, use the layers directly.
3833          */
3834         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3835                 return dev_flow->handle->layers;
3836         /* Convert L3 layers with decap action. */
3837         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3838                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3839         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3840                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3841         /* Convert L4 layers with decap action.  */
3842         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3843                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3844         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3845                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3846         return layers;
3847 }
3848
3849 /**
3850  * Get metadata split action information.
3851  *
3852  * @param[in] actions
3853  *   Pointer to the list of actions.
3854  * @param[out] qrss
3855  *   Pointer to the return pointer.
3856  * @param[out] qrss_type
3857  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3858  *   if no QUEUE/RSS is found.
3859  * @param[out] encap_idx
3860  *   Pointer to the index of the encap action if exists, otherwise the last
3861  *   action index.
3862  *
3863  * @return
3864  *   Total number of actions.
3865  */
3866 static int
3867 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3868                                        const struct rte_flow_action **qrss,
3869                                        int *encap_idx)
3870 {
3871         const struct rte_flow_action_raw_encap *raw_encap;
3872         int actions_n = 0;
3873         int raw_decap_idx = -1;
3874
3875         *encap_idx = -1;
3876         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3877                 switch (actions->type) {
3878                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3879                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3880                         *encap_idx = actions_n;
3881                         break;
3882                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3883                         raw_decap_idx = actions_n;
3884                         break;
3885                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3886                         raw_encap = actions->conf;
3887                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3888                                 *encap_idx = raw_decap_idx != -1 ?
3889                                                       raw_decap_idx : actions_n;
3890                         break;
3891                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3892                 case RTE_FLOW_ACTION_TYPE_RSS:
3893                         *qrss = actions;
3894                         break;
3895                 default:
3896                         break;
3897                 }
3898                 actions_n++;
3899         }
3900         if (*encap_idx == -1)
3901                 *encap_idx = actions_n;
3902         /* Count RTE_FLOW_ACTION_TYPE_END. */
3903         return actions_n + 1;
3904 }
3905
3906 /**
3907  * Check if the action will change packet.
3908  *
3909  * @param dev
3910  *   Pointer to Ethernet device.
3911  * @param[in] type
3912  *   action type.
3913  *
3914  * @return
3915  *   true if action will change packet, false otherwise.
3916  */
3917 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3918                                           enum rte_flow_action_type type)
3919 {
3920         struct mlx5_priv *priv = dev->data->dev_private;
3921
3922         switch (type) {
3923         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3924         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3925         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3926         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3927         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3928         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3929         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3930         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3931         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3932         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3933         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3934         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3935         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3936         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3937         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3938         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3939         case RTE_FLOW_ACTION_TYPE_SET_META:
3940         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3941         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3942         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3943         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3944         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3945         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3946         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3947         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3948         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3949         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3950         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3951         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3952                 return true;
3953         case RTE_FLOW_ACTION_TYPE_FLAG:
3954         case RTE_FLOW_ACTION_TYPE_MARK:
3955                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3956                         return true;
3957                 else
3958                         return false;
3959         default:
3960                 return false;
3961         }
3962 }
3963
3964 /**
3965  * Check meter action from the action list.
3966  *
3967  * @param dev
3968  *   Pointer to Ethernet device.
3969  * @param[in] actions
3970  *   Pointer to the list of actions.
3971  * @param[out] has_mtr
3972  *   Pointer to the meter exist flag.
3973  * @param[out] has_modify
3974  *   Pointer to the flag showing there's packet change action.
3975  * @param[out] meter_id
3976  *   Pointer to the meter id.
3977  *
3978  * @return
3979  *   Total number of actions.
3980  */
3981 static int
3982 flow_check_meter_action(struct rte_eth_dev *dev,
3983                         const struct rte_flow_action actions[],
3984                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3985 {
3986         const struct rte_flow_action_meter *mtr = NULL;
3987         int actions_n = 0;
3988
3989         MLX5_ASSERT(has_mtr);
3990         *has_mtr = false;
3991         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3992                 switch (actions->type) {
3993                 case RTE_FLOW_ACTION_TYPE_METER:
3994                         mtr = actions->conf;
3995                         *meter_id = mtr->mtr_id;
3996                         *has_mtr = true;
3997                         break;
3998                 default:
3999                         break;
4000                 }
4001                 if (!*has_mtr)
4002                         *has_modify |= flow_check_modify_action_type(dev,
4003                                                                 actions->type);
4004                 actions_n++;
4005         }
4006         /* Count RTE_FLOW_ACTION_TYPE_END. */
4007         return actions_n + 1;
4008 }
4009
4010 /**
4011  * Check if the flow should be split due to hairpin.
4012  * The reason for the split is that in current HW we can't
4013  * support encap and push-vlan on Rx, so if a flow contains
4014  * these actions we move it to Tx.
4015  *
4016  * @param dev
4017  *   Pointer to Ethernet device.
4018  * @param[in] attr
4019  *   Flow rule attributes.
4020  * @param[in] actions
4021  *   Associated actions (list terminated by the END action).
4022  *
4023  * @return
4024  *   > 0 the number of actions and the flow should be split,
4025  *   0 when no split required.
4026  */
4027 static int
4028 flow_check_hairpin_split(struct rte_eth_dev *dev,
4029                          const struct rte_flow_attr *attr,
4030                          const struct rte_flow_action actions[])
4031 {
4032         int queue_action = 0;
4033         int action_n = 0;
4034         int split = 0;
4035         const struct rte_flow_action_queue *queue;
4036         const struct rte_flow_action_rss *rss;
4037         const struct rte_flow_action_raw_encap *raw_encap;
4038         const struct rte_eth_hairpin_conf *conf;
4039
4040         if (!attr->ingress)
4041                 return 0;
4042         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4043                 switch (actions->type) {
4044                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4045                         queue = actions->conf;
4046                         if (queue == NULL)
4047                                 return 0;
4048                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4049                         if (conf == NULL || conf->tx_explicit != 0)
4050                                 return 0;
4051                         queue_action = 1;
4052                         action_n++;
4053                         break;
4054                 case RTE_FLOW_ACTION_TYPE_RSS:
4055                         rss = actions->conf;
4056                         if (rss == NULL || rss->queue_num == 0)
4057                                 return 0;
4058                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4059                         if (conf == NULL || conf->tx_explicit != 0)
4060                                 return 0;
4061                         queue_action = 1;
4062                         action_n++;
4063                         break;
4064                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4065                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4066                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4067                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4068                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4069                         split++;
4070                         action_n++;
4071                         break;
4072                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4073                         raw_encap = actions->conf;
4074                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4075                                 split++;
4076                         action_n++;
4077                         break;
4078                 default:
4079                         action_n++;
4080                         break;
4081                 }
4082         }
4083         if (split && queue_action)
4084                 return action_n;
4085         return 0;
4086 }
4087
4088 /* Declare flow create/destroy prototype in advance. */
4089 static uint32_t
4090 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4091                  const struct rte_flow_attr *attr,
4092                  const struct rte_flow_item items[],
4093                  const struct rte_flow_action actions[],
4094                  bool external, struct rte_flow_error *error);
4095
4096 static void
4097 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4098                   uint32_t flow_idx);
4099
4100 int
4101 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4102                       struct mlx5_list_entry *entry, void *cb_ctx)
4103 {
4104         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4105         struct mlx5_flow_mreg_copy_resource *mcp_res =
4106                                container_of(entry, typeof(*mcp_res), hlist_ent);
4107
4108         return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4109 }
4110
4111 struct mlx5_list_entry *
4112 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4113 {
4114         struct rte_eth_dev *dev = tool_ctx;
4115         struct mlx5_priv *priv = dev->data->dev_private;
4116         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4117         struct mlx5_flow_mreg_copy_resource *mcp_res;
4118         struct rte_flow_error *error = ctx->error;
4119         uint32_t idx = 0;
4120         int ret;
4121         uint32_t mark_id = *(uint32_t *)(ctx->data);
4122         struct rte_flow_attr attr = {
4123                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4124                 .ingress = 1,
4125         };
4126         struct mlx5_rte_flow_item_tag tag_spec = {
4127                 .data = mark_id,
4128         };
4129         struct rte_flow_item items[] = {
4130                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4131         };
4132         struct rte_flow_action_mark ftag = {
4133                 .id = mark_id,
4134         };
4135         struct mlx5_flow_action_copy_mreg cp_mreg = {
4136                 .dst = REG_B,
4137                 .src = REG_NON,
4138         };
4139         struct rte_flow_action_jump jump = {
4140                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4141         };
4142         struct rte_flow_action actions[] = {
4143                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4144         };
4145
4146         /* Fill the register fileds in the flow. */
4147         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4148         if (ret < 0)
4149                 return NULL;
4150         tag_spec.id = ret;
4151         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4152         if (ret < 0)
4153                 return NULL;
4154         cp_mreg.src = ret;
4155         /* Provide the full width of FLAG specific value. */
4156         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4157                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4158         /* Build a new flow. */
4159         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4160                 items[0] = (struct rte_flow_item){
4161                         .type = (enum rte_flow_item_type)
4162                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4163                         .spec = &tag_spec,
4164                 };
4165                 items[1] = (struct rte_flow_item){
4166                         .type = RTE_FLOW_ITEM_TYPE_END,
4167                 };
4168                 actions[0] = (struct rte_flow_action){
4169                         .type = (enum rte_flow_action_type)
4170                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4171                         .conf = &ftag,
4172                 };
4173                 actions[1] = (struct rte_flow_action){
4174                         .type = (enum rte_flow_action_type)
4175                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4176                         .conf = &cp_mreg,
4177                 };
4178                 actions[2] = (struct rte_flow_action){
4179                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4180                         .conf = &jump,
4181                 };
4182                 actions[3] = (struct rte_flow_action){
4183                         .type = RTE_FLOW_ACTION_TYPE_END,
4184                 };
4185         } else {
4186                 /* Default rule, wildcard match. */
4187                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4188                 items[0] = (struct rte_flow_item){
4189                         .type = RTE_FLOW_ITEM_TYPE_END,
4190                 };
4191                 actions[0] = (struct rte_flow_action){
4192                         .type = (enum rte_flow_action_type)
4193                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4194                         .conf = &cp_mreg,
4195                 };
4196                 actions[1] = (struct rte_flow_action){
4197                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4198                         .conf = &jump,
4199                 };
4200                 actions[2] = (struct rte_flow_action){
4201                         .type = RTE_FLOW_ACTION_TYPE_END,
4202                 };
4203         }
4204         /* Build a new entry. */
4205         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4206         if (!mcp_res) {
4207                 rte_errno = ENOMEM;
4208                 return NULL;
4209         }
4210         mcp_res->idx = idx;
4211         mcp_res->mark_id = mark_id;
4212         /*
4213          * The copy Flows are not included in any list. There
4214          * ones are referenced from other Flows and can not
4215          * be applied, removed, deleted in ardbitrary order
4216          * by list traversing.
4217          */
4218         mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4219                                         &attr, items, actions, false, error);
4220         if (!mcp_res->rix_flow) {
4221                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4222                 return NULL;
4223         }
4224         return &mcp_res->hlist_ent;
4225 }
4226
4227 struct mlx5_list_entry *
4228 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4229                       void *cb_ctx __rte_unused)
4230 {
4231         struct rte_eth_dev *dev = tool_ctx;
4232         struct mlx5_priv *priv = dev->data->dev_private;
4233         struct mlx5_flow_mreg_copy_resource *mcp_res;
4234         uint32_t idx = 0;
4235
4236         mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4237         if (!mcp_res) {
4238                 rte_errno = ENOMEM;
4239                 return NULL;
4240         }
4241         memcpy(mcp_res, oentry, sizeof(*mcp_res));
4242         mcp_res->idx = idx;
4243         return &mcp_res->hlist_ent;
4244 }
4245
4246 void
4247 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4248 {
4249         struct mlx5_flow_mreg_copy_resource *mcp_res =
4250                                container_of(entry, typeof(*mcp_res), hlist_ent);
4251         struct rte_eth_dev *dev = tool_ctx;
4252         struct mlx5_priv *priv = dev->data->dev_private;
4253
4254         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4255 }
4256
4257 /**
4258  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4259  *
4260  * As mark_id is unique, if there's already a registered flow for the mark_id,
4261  * return by increasing the reference counter of the resource. Otherwise, create
4262  * the resource (mcp_res) and flow.
4263  *
4264  * Flow looks like,
4265  *   - If ingress port is ANY and reg_c[1] is mark_id,
4266  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4267  *
4268  * For default flow (zero mark_id), flow is like,
4269  *   - If ingress port is ANY,
4270  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4271  *
4272  * @param dev
4273  *   Pointer to Ethernet device.
4274  * @param mark_id
4275  *   ID of MARK action, zero means default flow for META.
4276  * @param[out] error
4277  *   Perform verbose error reporting if not NULL.
4278  *
4279  * @return
4280  *   Associated resource on success, NULL otherwise and rte_errno is set.
4281  */
4282 static struct mlx5_flow_mreg_copy_resource *
4283 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4284                           struct rte_flow_error *error)
4285 {
4286         struct mlx5_priv *priv = dev->data->dev_private;
4287         struct mlx5_list_entry *entry;
4288         struct mlx5_flow_cb_ctx ctx = {
4289                 .dev = dev,
4290                 .error = error,
4291                 .data = &mark_id,
4292         };
4293
4294         /* Check if already registered. */
4295         MLX5_ASSERT(priv->mreg_cp_tbl);
4296         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4297         if (!entry)
4298                 return NULL;
4299         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4300                             hlist_ent);
4301 }
4302
4303 void
4304 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4305 {
4306         struct mlx5_flow_mreg_copy_resource *mcp_res =
4307                                container_of(entry, typeof(*mcp_res), hlist_ent);
4308         struct rte_eth_dev *dev = tool_ctx;
4309         struct mlx5_priv *priv = dev->data->dev_private;
4310
4311         MLX5_ASSERT(mcp_res->rix_flow);
4312         flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4313         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4314 }
4315
4316 /**
4317  * Release flow in RX_CP_TBL.
4318  *
4319  * @param dev
4320  *   Pointer to Ethernet device.
4321  * @flow
4322  *   Parent flow for wich copying is provided.
4323  */
4324 static void
4325 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4326                           struct rte_flow *flow)
4327 {
4328         struct mlx5_flow_mreg_copy_resource *mcp_res;
4329         struct mlx5_priv *priv = dev->data->dev_private;
4330
4331         if (!flow->rix_mreg_copy)
4332                 return;
4333         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4334                                  flow->rix_mreg_copy);
4335         if (!mcp_res || !priv->mreg_cp_tbl)
4336                 return;
4337         MLX5_ASSERT(mcp_res->rix_flow);
4338         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4339         flow->rix_mreg_copy = 0;
4340 }
4341
4342 /**
4343  * Remove the default copy action from RX_CP_TBL.
4344  *
4345  * This functions is called in the mlx5_dev_start(). No thread safe
4346  * is guaranteed.
4347  *
4348  * @param dev
4349  *   Pointer to Ethernet device.
4350  */
4351 static void
4352 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4353 {
4354         struct mlx5_list_entry *entry;
4355         struct mlx5_priv *priv = dev->data->dev_private;
4356         struct mlx5_flow_cb_ctx ctx;
4357         uint32_t mark_id;
4358
4359         /* Check if default flow is registered. */
4360         if (!priv->mreg_cp_tbl)
4361                 return;
4362         mark_id = MLX5_DEFAULT_COPY_ID;
4363         ctx.data = &mark_id;
4364         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4365         if (!entry)
4366                 return;
4367         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4368 }
4369
4370 /**
4371  * Add the default copy action in in RX_CP_TBL.
4372  *
4373  * This functions is called in the mlx5_dev_start(). No thread safe
4374  * is guaranteed.
4375  *
4376  * @param dev
4377  *   Pointer to Ethernet device.
4378  * @param[out] error
4379  *   Perform verbose error reporting if not NULL.
4380  *
4381  * @return
4382  *   0 for success, negative value otherwise and rte_errno is set.
4383  */
4384 static int
4385 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4386                                   struct rte_flow_error *error)
4387 {
4388         struct mlx5_priv *priv = dev->data->dev_private;
4389         struct mlx5_flow_mreg_copy_resource *mcp_res;
4390         struct mlx5_flow_cb_ctx ctx;
4391         uint32_t mark_id;
4392
4393         /* Check whether extensive metadata feature is engaged. */
4394         if (!priv->config.dv_flow_en ||
4395             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4396             !mlx5_flow_ext_mreg_supported(dev) ||
4397             !priv->sh->dv_regc0_mask)
4398                 return 0;
4399         /*
4400          * Add default mreg copy flow may be called multiple time, but
4401          * only be called once in stop. Avoid register it twice.
4402          */
4403         mark_id = MLX5_DEFAULT_COPY_ID;
4404         ctx.data = &mark_id;
4405         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4406                 return 0;
4407         mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4408         if (!mcp_res)
4409                 return -rte_errno;
4410         return 0;
4411 }
4412
4413 /**
4414  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4415  *
4416  * All the flow having Q/RSS action should be split by
4417  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4418  * performs the following,
4419  *   - CQE->flow_tag := reg_c[1] (MARK)
4420  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4421  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4422  * but there should be a flow per each MARK ID set by MARK action.
4423  *
4424  * For the aforementioned reason, if there's a MARK action in flow's action
4425  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4426  * the MARK ID to CQE's flow_tag like,
4427  *   - If reg_c[1] is mark_id,
4428  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4429  *
4430  * For SET_META action which stores value in reg_c[0], as the destination is
4431  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4432  * MARK ID means the default flow. The default flow looks like,
4433  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4434  *
4435  * @param dev
4436  *   Pointer to Ethernet device.
4437  * @param flow
4438  *   Pointer to flow structure.
4439  * @param[in] actions
4440  *   Pointer to the list of actions.
4441  * @param[out] error
4442  *   Perform verbose error reporting if not NULL.
4443  *
4444  * @return
4445  *   0 on success, negative value otherwise and rte_errno is set.
4446  */
4447 static int
4448 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4449                             struct rte_flow *flow,
4450                             const struct rte_flow_action *actions,
4451                             struct rte_flow_error *error)
4452 {
4453         struct mlx5_priv *priv = dev->data->dev_private;
4454         struct mlx5_dev_config *config = &priv->config;
4455         struct mlx5_flow_mreg_copy_resource *mcp_res;
4456         const struct rte_flow_action_mark *mark;
4457
4458         /* Check whether extensive metadata feature is engaged. */
4459         if (!config->dv_flow_en ||
4460             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4461             !mlx5_flow_ext_mreg_supported(dev) ||
4462             !priv->sh->dv_regc0_mask)
4463                 return 0;
4464         /* Find MARK action. */
4465         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4466                 switch (actions->type) {
4467                 case RTE_FLOW_ACTION_TYPE_FLAG:
4468                         mcp_res = flow_mreg_add_copy_action
4469                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4470                         if (!mcp_res)
4471                                 return -rte_errno;
4472                         flow->rix_mreg_copy = mcp_res->idx;
4473                         return 0;
4474                 case RTE_FLOW_ACTION_TYPE_MARK:
4475                         mark = (const struct rte_flow_action_mark *)
4476                                 actions->conf;
4477                         mcp_res =
4478                                 flow_mreg_add_copy_action(dev, mark->id, error);
4479                         if (!mcp_res)
4480                                 return -rte_errno;
4481                         flow->rix_mreg_copy = mcp_res->idx;
4482                         return 0;
4483                 default:
4484                         break;
4485                 }
4486         }
4487         return 0;
4488 }
4489
4490 #define MLX5_MAX_SPLIT_ACTIONS 24
4491 #define MLX5_MAX_SPLIT_ITEMS 24
4492
4493 /**
4494  * Split the hairpin flow.
4495  * Since HW can't support encap and push-vlan on Rx, we move these
4496  * actions to Tx.
4497  * If the count action is after the encap then we also
4498  * move the count action. in this case the count will also measure
4499  * the outer bytes.
4500  *
4501  * @param dev
4502  *   Pointer to Ethernet device.
4503  * @param[in] actions
4504  *   Associated actions (list terminated by the END action).
4505  * @param[out] actions_rx
4506  *   Rx flow actions.
4507  * @param[out] actions_tx
4508  *   Tx flow actions..
4509  * @param[out] pattern_tx
4510  *   The pattern items for the Tx flow.
4511  * @param[out] flow_id
4512  *   The flow ID connected to this flow.
4513  *
4514  * @return
4515  *   0 on success.
4516  */
4517 static int
4518 flow_hairpin_split(struct rte_eth_dev *dev,
4519                    const struct rte_flow_action actions[],
4520                    struct rte_flow_action actions_rx[],
4521                    struct rte_flow_action actions_tx[],
4522                    struct rte_flow_item pattern_tx[],
4523                    uint32_t flow_id)
4524 {
4525         const struct rte_flow_action_raw_encap *raw_encap;
4526         const struct rte_flow_action_raw_decap *raw_decap;
4527         struct mlx5_rte_flow_action_set_tag *set_tag;
4528         struct rte_flow_action *tag_action;
4529         struct mlx5_rte_flow_item_tag *tag_item;
4530         struct rte_flow_item *item;
4531         char *addr;
4532         int encap = 0;
4533
4534         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4535                 switch (actions->type) {
4536                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4537                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4538                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4539                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4540                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4541                         rte_memcpy(actions_tx, actions,
4542                                sizeof(struct rte_flow_action));
4543                         actions_tx++;
4544                         break;
4545                 case RTE_FLOW_ACTION_TYPE_COUNT:
4546                         if (encap) {
4547                                 rte_memcpy(actions_tx, actions,
4548                                            sizeof(struct rte_flow_action));
4549                                 actions_tx++;
4550                         } else {
4551                                 rte_memcpy(actions_rx, actions,
4552                                            sizeof(struct rte_flow_action));
4553                                 actions_rx++;
4554                         }
4555                         break;
4556                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4557                         raw_encap = actions->conf;
4558                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4559                                 memcpy(actions_tx, actions,
4560                                        sizeof(struct rte_flow_action));
4561                                 actions_tx++;
4562                                 encap = 1;
4563                         } else {
4564                                 rte_memcpy(actions_rx, actions,
4565                                            sizeof(struct rte_flow_action));
4566                                 actions_rx++;
4567                         }
4568                         break;
4569                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4570                         raw_decap = actions->conf;
4571                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4572                                 memcpy(actions_tx, actions,
4573                                        sizeof(struct rte_flow_action));
4574                                 actions_tx++;
4575                         } else {
4576                                 rte_memcpy(actions_rx, actions,
4577                                            sizeof(struct rte_flow_action));
4578                                 actions_rx++;
4579                         }
4580                         break;
4581                 default:
4582                         rte_memcpy(actions_rx, actions,
4583                                    sizeof(struct rte_flow_action));
4584                         actions_rx++;
4585                         break;
4586                 }
4587         }
4588         /* Add set meta action and end action for the Rx flow. */
4589         tag_action = actions_rx;
4590         tag_action->type = (enum rte_flow_action_type)
4591                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4592         actions_rx++;
4593         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4594         actions_rx++;
4595         set_tag = (void *)actions_rx;
4596         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4597                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4598                 .data = flow_id,
4599         };
4600         MLX5_ASSERT(set_tag->id > REG_NON);
4601         tag_action->conf = set_tag;
4602         /* Create Tx item list. */
4603         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4604         addr = (void *)&pattern_tx[2];
4605         item = pattern_tx;
4606         item->type = (enum rte_flow_item_type)
4607                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4608         tag_item = (void *)addr;
4609         tag_item->data = flow_id;
4610         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4611         MLX5_ASSERT(set_tag->id > REG_NON);
4612         item->spec = tag_item;
4613         addr += sizeof(struct mlx5_rte_flow_item_tag);
4614         tag_item = (void *)addr;
4615         tag_item->data = UINT32_MAX;
4616         tag_item->id = UINT16_MAX;
4617         item->mask = tag_item;
4618         item->last = NULL;
4619         item++;
4620         item->type = RTE_FLOW_ITEM_TYPE_END;
4621         return 0;
4622 }
4623
4624 /**
4625  * The last stage of splitting chain, just creates the subflow
4626  * without any modification.
4627  *
4628  * @param[in] dev
4629  *   Pointer to Ethernet device.
4630  * @param[in] flow
4631  *   Parent flow structure pointer.
4632  * @param[in, out] sub_flow
4633  *   Pointer to return the created subflow, may be NULL.
4634  * @param[in] attr
4635  *   Flow rule attributes.
4636  * @param[in] items
4637  *   Pattern specification (list terminated by the END pattern item).
4638  * @param[in] actions
4639  *   Associated actions (list terminated by the END action).
4640  * @param[in] flow_split_info
4641  *   Pointer to flow split info structure.
4642  * @param[out] error
4643  *   Perform verbose error reporting if not NULL.
4644  * @return
4645  *   0 on success, negative value otherwise
4646  */
4647 static int
4648 flow_create_split_inner(struct rte_eth_dev *dev,
4649                         struct rte_flow *flow,
4650                         struct mlx5_flow **sub_flow,
4651                         const struct rte_flow_attr *attr,
4652                         const struct rte_flow_item items[],
4653                         const struct rte_flow_action actions[],
4654                         struct mlx5_flow_split_info *flow_split_info,
4655                         struct rte_flow_error *error)
4656 {
4657         struct mlx5_flow *dev_flow;
4658
4659         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4660                                     flow_split_info->flow_idx, error);
4661         if (!dev_flow)
4662                 return -rte_errno;
4663         dev_flow->flow = flow;
4664         dev_flow->external = flow_split_info->external;
4665         dev_flow->skip_scale = flow_split_info->skip_scale;
4666         /* Subflow object was created, we must include one in the list. */
4667         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4668                       dev_flow->handle, next);
4669         /*
4670          * If dev_flow is as one of the suffix flow, some actions in suffix
4671          * flow may need some user defined item layer flags, and pass the
4672          * Metadate rxq mark flag to suffix flow as well.
4673          */
4674         if (flow_split_info->prefix_layers)
4675                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4676         if (flow_split_info->prefix_mark)
4677                 dev_flow->handle->mark = 1;
4678         if (sub_flow)
4679                 *sub_flow = dev_flow;
4680 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4681         dev_flow->dv.table_id = flow_split_info->table_id;
4682 #endif
4683         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4684 }
4685
4686 /**
4687  * Get the sub policy of a meter.
4688  *
4689  * @param[in] dev
4690  *   Pointer to Ethernet device.
4691  * @param[in] flow
4692  *   Parent flow structure pointer.
4693  * @param wks
4694  *   Pointer to thread flow work space.
4695  * @param[in] attr
4696  *   Flow rule attributes.
4697  * @param[in] items
4698  *   Pattern specification (list terminated by the END pattern item).
4699  * @param[out] error
4700  *   Perform verbose error reporting if not NULL.
4701  *
4702  * @return
4703  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4704  */
4705 static struct mlx5_flow_meter_sub_policy *
4706 get_meter_sub_policy(struct rte_eth_dev *dev,
4707                      struct rte_flow *flow,
4708                      struct mlx5_flow_workspace *wks,
4709                      const struct rte_flow_attr *attr,
4710                      const struct rte_flow_item items[],
4711                      struct rte_flow_error *error)
4712 {
4713         struct mlx5_flow_meter_policy *policy;
4714         struct mlx5_flow_meter_policy *final_policy;
4715         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4716
4717         policy = wks->policy;
4718         final_policy = policy->is_hierarchy ? wks->final_policy : policy;
4719         if (final_policy->is_rss || final_policy->is_queue) {
4720                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4721                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4722                 uint32_t i;
4723
4724                 /*
4725                  * This is a tmp dev_flow,
4726                  * no need to register any matcher for it in translate.
4727                  */
4728                 wks->skip_matcher_reg = 1;
4729                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4730                         struct mlx5_flow dev_flow = {0};
4731                         struct mlx5_flow_handle dev_handle = { {0} };
4732                         uint8_t fate = final_policy->act_cnt[i].fate_action;
4733
4734                         if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
4735                                 const struct rte_flow_action_rss *rss_act =
4736                                         final_policy->act_cnt[i].rss->conf;
4737                                 struct rte_flow_action rss_actions[2] = {
4738                                         [0] = {
4739                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4740                                         .conf = rss_act,
4741                                         },
4742                                         [1] = {
4743                                         .type = RTE_FLOW_ACTION_TYPE_END,
4744                                         .conf = NULL,
4745                                         }
4746                                 };
4747
4748                                 dev_flow.handle = &dev_handle;
4749                                 dev_flow.ingress = attr->ingress;
4750                                 dev_flow.flow = flow;
4751                                 dev_flow.external = 0;
4752 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4753                                 dev_flow.dv.transfer = attr->transfer;
4754 #endif
4755                                 /**
4756                                  * Translate RSS action to get rss hash fields.
4757                                  */
4758                                 if (flow_drv_translate(dev, &dev_flow, attr,
4759                                                 items, rss_actions, error))
4760                                         goto exit;
4761                                 rss_desc_v[i] = wks->rss_desc;
4762                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4763                                 rss_desc_v[i].hash_fields =
4764                                                 dev_flow.hash_fields;
4765                                 rss_desc_v[i].queue_num =
4766                                                 rss_desc_v[i].hash_fields ?
4767                                                 rss_desc_v[i].queue_num : 1;
4768                                 rss_desc_v[i].tunnel =
4769                                                 !!(dev_flow.handle->layers &
4770                                                    MLX5_FLOW_LAYER_TUNNEL);
4771                                 /* Use the RSS queues in the containers. */
4772                                 rss_desc_v[i].queue =
4773                                         (uint16_t *)(uintptr_t)rss_act->queue;
4774                                 rss_desc[i] = &rss_desc_v[i];
4775                         } else if (fate == MLX5_FLOW_FATE_QUEUE) {
4776                                 /* This is queue action. */
4777                                 rss_desc_v[i] = wks->rss_desc;
4778                                 rss_desc_v[i].key_len = 0;
4779                                 rss_desc_v[i].hash_fields = 0;
4780                                 rss_desc_v[i].queue =
4781                                         &final_policy->act_cnt[i].queue;
4782                                 rss_desc_v[i].queue_num = 1;
4783                                 rss_desc[i] = &rss_desc_v[i];
4784                         } else {
4785                                 rss_desc[i] = NULL;
4786                         }
4787                 }
4788                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4789                                                 flow, policy, rss_desc);
4790         } else {
4791                 enum mlx5_meter_domain mtr_domain =
4792                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4793                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4794                                                 MLX5_MTR_DOMAIN_INGRESS);
4795                 sub_policy = policy->sub_policys[mtr_domain][0];
4796         }
4797         if (!sub_policy)
4798                 rte_flow_error_set(error, EINVAL,
4799                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4800                                    "Failed to get meter sub-policy.");
4801 exit:
4802         return sub_policy;
4803 }
4804
4805 /**
4806  * Split the meter flow.
4807  *
4808  * As meter flow will split to three sub flow, other than meter
4809  * action, the other actions make sense to only meter accepts
4810  * the packet. If it need to be dropped, no other additional
4811  * actions should be take.
4812  *
4813  * One kind of special action which decapsulates the L3 tunnel
4814  * header will be in the prefix sub flow, as not to take the
4815  * L3 tunnel header into account.
4816  *
4817  * @param[in] dev
4818  *   Pointer to Ethernet device.
4819  * @param[in] flow
4820  *   Parent flow structure pointer.
4821  * @param wks
4822  *   Pointer to thread flow work space.
4823  * @param[in] attr
4824  *   Flow rule attributes.
4825  * @param[in] items
4826  *   Pattern specification (list terminated by the END pattern item).
4827  * @param[out] sfx_items
4828  *   Suffix flow match items (list terminated by the END pattern item).
4829  * @param[in] actions
4830  *   Associated actions (list terminated by the END action).
4831  * @param[out] actions_sfx
4832  *   Suffix flow actions.
4833  * @param[out] actions_pre
4834  *   Prefix flow actions.
4835  * @param[out] mtr_flow_id
4836  *   Pointer to meter flow id.
4837  * @param[out] error
4838  *   Perform verbose error reporting if not NULL.
4839  *
4840  * @return
4841  *   0 on success, a negative errno value otherwise and rte_errno is set.
4842  */
4843 static int
4844 flow_meter_split_prep(struct rte_eth_dev *dev,
4845                       struct rte_flow *flow,
4846                       struct mlx5_flow_workspace *wks,
4847                       const struct rte_flow_attr *attr,
4848                       const struct rte_flow_item items[],
4849                       struct rte_flow_item sfx_items[],
4850                       const struct rte_flow_action actions[],
4851                       struct rte_flow_action actions_sfx[],
4852                       struct rte_flow_action actions_pre[],
4853                       uint32_t *mtr_flow_id,
4854                       struct rte_flow_error *error)
4855 {
4856         struct mlx5_priv *priv = dev->data->dev_private;
4857         struct mlx5_flow_meter_info *fm = wks->fm;
4858         struct rte_flow_action *tag_action = NULL;
4859         struct rte_flow_item *tag_item;
4860         struct mlx5_rte_flow_action_set_tag *set_tag;
4861         const struct rte_flow_action_raw_encap *raw_encap;
4862         const struct rte_flow_action_raw_decap *raw_decap;
4863         struct mlx5_rte_flow_item_tag *tag_item_spec;
4864         struct mlx5_rte_flow_item_tag *tag_item_mask;
4865         uint32_t tag_id = 0;
4866         struct rte_flow_item *vlan_item_dst = NULL;
4867         const struct rte_flow_item *vlan_item_src = NULL;
4868         struct rte_flow_action *hw_mtr_action;
4869         struct rte_flow_action *action_pre_head = NULL;
4870         int32_t flow_src_port = priv->representor_id;
4871         bool mtr_first;
4872         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4873         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4874                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4875         uint32_t flow_id = 0;
4876         uint32_t flow_id_reversed = 0;
4877         uint8_t flow_id_bits = 0;
4878         int shift;
4879
4880         /* Prepare the suffix subflow items. */
4881         tag_item = sfx_items++;
4882         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4883                 struct mlx5_priv *port_priv;
4884                 const struct rte_flow_item_port_id *pid_v;
4885                 int item_type = items->type;
4886
4887                 switch (item_type) {
4888                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4889                         pid_v = items->spec;
4890                         MLX5_ASSERT(pid_v);
4891                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
4892                         if (!port_priv)
4893                                 return rte_flow_error_set(error,
4894                                                 rte_errno,
4895                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
4896                                                 pid_v,
4897                                                 "Failed to get port info.");
4898                         flow_src_port = port_priv->representor_id;
4899                         if (!fm->def_policy && wks->policy->is_hierarchy &&
4900                             flow_src_port != priv->representor_id) {
4901                                 if (flow_drv_mtr_hierarchy_rule_create(dev,
4902                                                                 flow, fm,
4903                                                                 flow_src_port,
4904                                                                 items,
4905                                                                 error))
4906                                         return -rte_errno;
4907                         }
4908                         memcpy(sfx_items, items, sizeof(*sfx_items));
4909                         sfx_items++;
4910                         break;
4911                 case RTE_FLOW_ITEM_TYPE_VLAN:
4912                         /* Determine if copy vlan item below. */
4913                         vlan_item_src = items;
4914                         vlan_item_dst = sfx_items++;
4915                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
4916                         break;
4917                 default:
4918                         break;
4919                 }
4920         }
4921         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4922         sfx_items++;
4923         mtr_first = priv->sh->meter_aso_en &&
4924                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
4925         /* For ASO meter, meter must be before tag in TX direction. */
4926         if (mtr_first) {
4927                 action_pre_head = actions_pre++;
4928                 /* Leave space for tag action. */
4929                 tag_action = actions_pre++;
4930         }
4931         /* Prepare the actions for prefix and suffix flow. */
4932         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4933                 struct rte_flow_action *action_cur = NULL;
4934
4935                 switch (actions->type) {
4936                 case RTE_FLOW_ACTION_TYPE_METER:
4937                         if (mtr_first) {
4938                                 action_cur = action_pre_head;
4939                         } else {
4940                                 /* Leave space for tag action. */
4941                                 tag_action = actions_pre++;
4942                                 action_cur = actions_pre++;
4943                         }
4944                         break;
4945                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4946                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4947                         action_cur = actions_pre++;
4948                         break;
4949                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4950                         raw_encap = actions->conf;
4951                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4952                                 action_cur = actions_pre++;
4953                         break;
4954                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4955                         raw_decap = actions->conf;
4956                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4957                                 action_cur = actions_pre++;
4958                         break;
4959                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4960                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4961                         if (vlan_item_dst && vlan_item_src) {
4962                                 memcpy(vlan_item_dst, vlan_item_src,
4963                                         sizeof(*vlan_item_dst));
4964                                 /*
4965                                  * Convert to internal match item, it is used
4966                                  * for vlan push and set vid.
4967                                  */
4968                                 vlan_item_dst->type = (enum rte_flow_item_type)
4969                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4970                         }
4971                         break;
4972                 default:
4973                         break;
4974                 }
4975                 if (!action_cur)
4976                         action_cur = (fm->def_policy) ?
4977                                         actions_sfx++ : actions_pre++;
4978                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4979         }
4980         /* Add end action to the actions. */
4981         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4982         if (priv->sh->meter_aso_en) {
4983                 /**
4984                  * For ASO meter, need to add an extra jump action explicitly,
4985                  * to jump from meter to policer table.
4986                  */
4987                 struct mlx5_flow_meter_sub_policy *sub_policy;
4988                 struct mlx5_flow_tbl_data_entry *tbl_data;
4989
4990                 if (!fm->def_policy) {
4991                         sub_policy = get_meter_sub_policy(dev, flow, wks,
4992                                                           attr, items, error);
4993                         if (!sub_policy)
4994                                 return -rte_errno;
4995                 } else {
4996                         enum mlx5_meter_domain mtr_domain =
4997                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4998                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4999                                                 MLX5_MTR_DOMAIN_INGRESS);
5000
5001                         sub_policy =
5002                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5003                 }
5004                 tbl_data = container_of(sub_policy->tbl_rsc,
5005                                         struct mlx5_flow_tbl_data_entry, tbl);
5006                 hw_mtr_action = actions_pre++;
5007                 hw_mtr_action->type = (enum rte_flow_action_type)
5008                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5009                 hw_mtr_action->conf = tbl_data->jump.action;
5010         }
5011         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5012         actions_pre++;
5013         if (!tag_action)
5014                 return rte_flow_error_set(error, ENOMEM,
5015                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5016                                           NULL, "No tag action space.");
5017         if (!mtr_flow_id) {
5018                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5019                 goto exit;
5020         }
5021         /* Only default-policy Meter creates mtr flow id. */
5022         if (fm->def_policy) {
5023                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5024                 if (!tag_id)
5025                         return rte_flow_error_set(error, ENOMEM,
5026                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5027                                         "Failed to allocate meter flow id.");
5028                 flow_id = tag_id - 1;
5029                 flow_id_bits = (!flow_id) ? 1 :
5030                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
5031                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5032                     mtr_reg_bits) {
5033                         mlx5_ipool_free(fm->flow_ipool, tag_id);
5034                         return rte_flow_error_set(error, EINVAL,
5035                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5036                                         "Meter flow id exceeds max limit.");
5037                 }
5038                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5039                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5040         }
5041         /* Build tag actions and items for meter_id/meter flow_id. */
5042         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5043         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5044         tag_item_mask = tag_item_spec + 1;
5045         /* Both flow_id and meter_id share the same register. */
5046         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5047                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5048                                                             0, error),
5049                 .offset = mtr_id_offset,
5050                 .length = mtr_reg_bits,
5051                 .data = flow->meter,
5052         };
5053         /*
5054          * The color Reg bits used by flow_id are growing from
5055          * msb to lsb, so must do bit reverse for flow_id val in RegC.
5056          */
5057         for (shift = 0; shift < flow_id_bits; shift++)
5058                 flow_id_reversed = (flow_id_reversed << 1) |
5059                                 ((flow_id >> shift) & 0x1);
5060         set_tag->data |=
5061                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5062         tag_item_spec->id = set_tag->id;
5063         tag_item_spec->data = set_tag->data << mtr_id_offset;
5064         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5065         tag_action->type = (enum rte_flow_action_type)
5066                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5067         tag_action->conf = set_tag;
5068         tag_item->type = (enum rte_flow_item_type)
5069                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5070         tag_item->spec = tag_item_spec;
5071         tag_item->last = NULL;
5072         tag_item->mask = tag_item_mask;
5073 exit:
5074         if (mtr_flow_id)
5075                 *mtr_flow_id = tag_id;
5076         return 0;
5077 }
5078
5079 /**
5080  * Split action list having QUEUE/RSS for metadata register copy.
5081  *
5082  * Once Q/RSS action is detected in user's action list, the flow action
5083  * should be split in order to copy metadata registers, which will happen in
5084  * RX_CP_TBL like,
5085  *   - CQE->flow_tag := reg_c[1] (MARK)
5086  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5087  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5088  * This is because the last action of each flow must be a terminal action
5089  * (QUEUE, RSS or DROP).
5090  *
5091  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5092  * stored and kept in the mlx5_flow structure per each sub_flow.
5093  *
5094  * The Q/RSS action is replaced with,
5095  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5096  * And the following JUMP action is added at the end,
5097  *   - JUMP, to RX_CP_TBL.
5098  *
5099  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5100  * flow_create_split_metadata() routine. The flow will look like,
5101  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5102  *
5103  * @param dev
5104  *   Pointer to Ethernet device.
5105  * @param[out] split_actions
5106  *   Pointer to store split actions to jump to CP_TBL.
5107  * @param[in] actions
5108  *   Pointer to the list of original flow actions.
5109  * @param[in] qrss
5110  *   Pointer to the Q/RSS action.
5111  * @param[in] actions_n
5112  *   Number of original actions.
5113  * @param[out] error
5114  *   Perform verbose error reporting if not NULL.
5115  *
5116  * @return
5117  *   non-zero unique flow_id on success, otherwise 0 and
5118  *   error/rte_error are set.
5119  */
5120 static uint32_t
5121 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5122                           struct rte_flow_action *split_actions,
5123                           const struct rte_flow_action *actions,
5124                           const struct rte_flow_action *qrss,
5125                           int actions_n, struct rte_flow_error *error)
5126 {
5127         struct mlx5_priv *priv = dev->data->dev_private;
5128         struct mlx5_rte_flow_action_set_tag *set_tag;
5129         struct rte_flow_action_jump *jump;
5130         const int qrss_idx = qrss - actions;
5131         uint32_t flow_id = 0;
5132         int ret = 0;
5133
5134         /*
5135          * Given actions will be split
5136          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5137          * - Add jump to mreg CP_TBL.
5138          * As a result, there will be one more action.
5139          */
5140         ++actions_n;
5141         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5142         set_tag = (void *)(split_actions + actions_n);
5143         /*
5144          * If tag action is not set to void(it means we are not the meter
5145          * suffix flow), add the tag action. Since meter suffix flow already
5146          * has the tag added.
5147          */
5148         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
5149                 /*
5150                  * Allocate the new subflow ID. This one is unique within
5151                  * device and not shared with representors. Otherwise,
5152                  * we would have to resolve multi-thread access synch
5153                  * issue. Each flow on the shared device is appended
5154                  * with source vport identifier, so the resulting
5155                  * flows will be unique in the shared (by master and
5156                  * representors) domain even if they have coinciding
5157                  * IDs.
5158                  */
5159                 mlx5_ipool_malloc(priv->sh->ipool
5160                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5161                 if (!flow_id)
5162                         return rte_flow_error_set(error, ENOMEM,
5163                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5164                                                   NULL, "can't allocate id "
5165                                                   "for split Q/RSS subflow");
5166                 /* Internal SET_TAG action to set flow ID. */
5167                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5168                         .data = flow_id,
5169                 };
5170                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5171                 if (ret < 0)
5172                         return ret;
5173                 set_tag->id = ret;
5174                 /* Construct new actions array. */
5175                 /* Replace QUEUE/RSS action. */
5176                 split_actions[qrss_idx] = (struct rte_flow_action){
5177                         .type = (enum rte_flow_action_type)
5178                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5179                         .conf = set_tag,
5180                 };
5181         }
5182         /* JUMP action to jump to mreg copy table (CP_TBL). */
5183         jump = (void *)(set_tag + 1);
5184         *jump = (struct rte_flow_action_jump){
5185                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5186         };
5187         split_actions[actions_n - 2] = (struct rte_flow_action){
5188                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5189                 .conf = jump,
5190         };
5191         split_actions[actions_n - 1] = (struct rte_flow_action){
5192                 .type = RTE_FLOW_ACTION_TYPE_END,
5193         };
5194         return flow_id;
5195 }
5196
5197 /**
5198  * Extend the given action list for Tx metadata copy.
5199  *
5200  * Copy the given action list to the ext_actions and add flow metadata register
5201  * copy action in order to copy reg_a set by WQE to reg_c[0].
5202  *
5203  * @param[out] ext_actions
5204  *   Pointer to the extended action list.
5205  * @param[in] actions
5206  *   Pointer to the list of actions.
5207  * @param[in] actions_n
5208  *   Number of actions in the list.
5209  * @param[out] error
5210  *   Perform verbose error reporting if not NULL.
5211  * @param[in] encap_idx
5212  *   The encap action inndex.
5213  *
5214  * @return
5215  *   0 on success, negative value otherwise
5216  */
5217 static int
5218 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5219                        struct rte_flow_action *ext_actions,
5220                        const struct rte_flow_action *actions,
5221                        int actions_n, struct rte_flow_error *error,
5222                        int encap_idx)
5223 {
5224         struct mlx5_flow_action_copy_mreg *cp_mreg =
5225                 (struct mlx5_flow_action_copy_mreg *)
5226                         (ext_actions + actions_n + 1);
5227         int ret;
5228
5229         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5230         if (ret < 0)
5231                 return ret;
5232         cp_mreg->dst = ret;
5233         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5234         if (ret < 0)
5235                 return ret;
5236         cp_mreg->src = ret;
5237         if (encap_idx != 0)
5238                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5239         if (encap_idx == actions_n - 1) {
5240                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5241                         .type = (enum rte_flow_action_type)
5242                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5243                         .conf = cp_mreg,
5244                 };
5245                 ext_actions[actions_n] = (struct rte_flow_action){
5246                         .type = RTE_FLOW_ACTION_TYPE_END,
5247                 };
5248         } else {
5249                 ext_actions[encap_idx] = (struct rte_flow_action){
5250                         .type = (enum rte_flow_action_type)
5251                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5252                         .conf = cp_mreg,
5253                 };
5254                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5255                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5256         }
5257         return 0;
5258 }
5259
5260 /**
5261  * Check the match action from the action list.
5262  *
5263  * @param[in] actions
5264  *   Pointer to the list of actions.
5265  * @param[in] attr
5266  *   Flow rule attributes.
5267  * @param[in] action
5268  *   The action to be check if exist.
5269  * @param[out] match_action_pos
5270  *   Pointer to the position of the matched action if exists, otherwise is -1.
5271  * @param[out] qrss_action_pos
5272  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5273  * @param[out] modify_after_mirror
5274  *   Pointer to the flag of modify action after FDB mirroring.
5275  *
5276  * @return
5277  *   > 0 the total number of actions.
5278  *   0 if not found match action in action list.
5279  */
5280 static int
5281 flow_check_match_action(const struct rte_flow_action actions[],
5282                         const struct rte_flow_attr *attr,
5283                         enum rte_flow_action_type action,
5284                         int *match_action_pos, int *qrss_action_pos,
5285                         int *modify_after_mirror)
5286 {
5287         const struct rte_flow_action_sample *sample;
5288         const struct rte_flow_action_raw_decap *decap;
5289         int actions_n = 0;
5290         uint32_t ratio = 0;
5291         int sub_type = 0;
5292         int flag = 0;
5293         int fdb_mirror = 0;
5294
5295         *match_action_pos = -1;
5296         *qrss_action_pos = -1;
5297         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5298                 if (actions->type == action) {
5299                         flag = 1;
5300                         *match_action_pos = actions_n;
5301                 }
5302                 switch (actions->type) {
5303                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5304                 case RTE_FLOW_ACTION_TYPE_RSS:
5305                         *qrss_action_pos = actions_n;
5306                         break;
5307                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5308                         sample = actions->conf;
5309                         ratio = sample->ratio;
5310                         sub_type = ((const struct rte_flow_action *)
5311                                         (sample->actions))->type;
5312                         if (ratio == 1 && attr->transfer)
5313                                 fdb_mirror = 1;
5314                         break;
5315                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5316                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5317                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5318                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5319                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5320                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5321                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5322                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5323                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5324                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5325                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5326                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5327                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5328                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5329                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5330                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5331                 case RTE_FLOW_ACTION_TYPE_FLAG:
5332                 case RTE_FLOW_ACTION_TYPE_MARK:
5333                 case RTE_FLOW_ACTION_TYPE_SET_META:
5334                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5335                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5336                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5337                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5338                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5339                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5340                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5341                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5342                 case RTE_FLOW_ACTION_TYPE_METER:
5343                         if (fdb_mirror)
5344                                 *modify_after_mirror = 1;
5345                         break;
5346                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5347                         decap = actions->conf;
5348                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5349                                 ;
5350                         actions_n++;
5351                         if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5352                                 const struct rte_flow_action_raw_encap *encap =
5353                                                                 actions->conf;
5354                                 if (decap->size <=
5355                                         MLX5_ENCAPSULATION_DECISION_SIZE &&
5356                                     encap->size >
5357                                         MLX5_ENCAPSULATION_DECISION_SIZE)
5358                                         /* L3 encap. */
5359                                         break;
5360                         }
5361                         if (fdb_mirror)
5362                                 *modify_after_mirror = 1;
5363                         break;
5364                 default:
5365                         break;
5366                 }
5367                 actions_n++;
5368         }
5369         if (flag && fdb_mirror && !*modify_after_mirror) {
5370                 /* FDB mirroring uses the destination array to implement
5371                  * instead of FLOW_SAMPLER object.
5372                  */
5373                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5374                         flag = 0;
5375         }
5376         /* Count RTE_FLOW_ACTION_TYPE_END. */
5377         return flag ? actions_n + 1 : 0;
5378 }
5379
5380 #define SAMPLE_SUFFIX_ITEM 2
5381
5382 /**
5383  * Split the sample flow.
5384  *
5385  * As sample flow will split to two sub flow, sample flow with
5386  * sample action, the other actions will move to new suffix flow.
5387  *
5388  * Also add unique tag id with tag action in the sample flow,
5389  * the same tag id will be as match in the suffix flow.
5390  *
5391  * @param dev
5392  *   Pointer to Ethernet device.
5393  * @param[in] add_tag
5394  *   Add extra tag action flag.
5395  * @param[out] sfx_items
5396  *   Suffix flow match items (list terminated by the END pattern item).
5397  * @param[in] actions
5398  *   Associated actions (list terminated by the END action).
5399  * @param[out] actions_sfx
5400  *   Suffix flow actions.
5401  * @param[out] actions_pre
5402  *   Prefix flow actions.
5403  * @param[in] actions_n
5404  *  The total number of actions.
5405  * @param[in] sample_action_pos
5406  *   The sample action position.
5407  * @param[in] qrss_action_pos
5408  *   The Queue/RSS action position.
5409  * @param[in] jump_table
5410  *   Add extra jump action flag.
5411  * @param[out] error
5412  *   Perform verbose error reporting if not NULL.
5413  *
5414  * @return
5415  *   0 on success, or unique flow_id, a negative errno value
5416  *   otherwise and rte_errno is set.
5417  */
5418 static int
5419 flow_sample_split_prep(struct rte_eth_dev *dev,
5420                        int add_tag,
5421                        struct rte_flow_item sfx_items[],
5422                        const struct rte_flow_action actions[],
5423                        struct rte_flow_action actions_sfx[],
5424                        struct rte_flow_action actions_pre[],
5425                        int actions_n,
5426                        int sample_action_pos,
5427                        int qrss_action_pos,
5428                        int jump_table,
5429                        struct rte_flow_error *error)
5430 {
5431         struct mlx5_priv *priv = dev->data->dev_private;
5432         struct mlx5_rte_flow_action_set_tag *set_tag;
5433         struct mlx5_rte_flow_item_tag *tag_spec;
5434         struct mlx5_rte_flow_item_tag *tag_mask;
5435         struct rte_flow_action_jump *jump_action;
5436         uint32_t tag_id = 0;
5437         int index;
5438         int append_index = 0;
5439         int ret;
5440
5441         if (sample_action_pos < 0)
5442                 return rte_flow_error_set(error, EINVAL,
5443                                           RTE_FLOW_ERROR_TYPE_ACTION,
5444                                           NULL, "invalid position of sample "
5445                                           "action in list");
5446         /* Prepare the actions for prefix and suffix flow. */
5447         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5448                 index = qrss_action_pos;
5449                 /* Put the preceding the Queue/RSS action into prefix flow. */
5450                 if (index != 0)
5451                         memcpy(actions_pre, actions,
5452                                sizeof(struct rte_flow_action) * index);
5453                 /* Put others preceding the sample action into prefix flow. */
5454                 if (sample_action_pos > index + 1)
5455                         memcpy(actions_pre + index, actions + index + 1,
5456                                sizeof(struct rte_flow_action) *
5457                                (sample_action_pos - index - 1));
5458                 index = sample_action_pos - 1;
5459                 /* Put Queue/RSS action into Suffix flow. */
5460                 memcpy(actions_sfx, actions + qrss_action_pos,
5461                        sizeof(struct rte_flow_action));
5462                 actions_sfx++;
5463         } else {
5464                 index = sample_action_pos;
5465                 if (index != 0)
5466                         memcpy(actions_pre, actions,
5467                                sizeof(struct rte_flow_action) * index);
5468         }
5469         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5470          * For CX6DX and above, metadata registers Cx preserve their value,
5471          * add an extra tag action for NIC-RX and E-Switch Domain.
5472          */
5473         if (add_tag) {
5474                 /* Prepare the prefix tag action. */
5475                 append_index++;
5476                 set_tag = (void *)(actions_pre + actions_n + append_index);
5477                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5478                 if (ret < 0)
5479                         return ret;
5480                 mlx5_ipool_malloc(priv->sh->ipool
5481                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5482                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5483                         .id = ret,
5484                         .data = tag_id,
5485                 };
5486                 /* Prepare the suffix subflow items. */
5487                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5488                 tag_spec->data = tag_id;
5489                 tag_spec->id = set_tag->id;
5490                 tag_mask = tag_spec + 1;
5491                 tag_mask->data = UINT32_MAX;
5492                 sfx_items[0] = (struct rte_flow_item){
5493                         .type = (enum rte_flow_item_type)
5494                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5495                         .spec = tag_spec,
5496                         .last = NULL,
5497                         .mask = tag_mask,
5498                 };
5499                 sfx_items[1] = (struct rte_flow_item){
5500                         .type = (enum rte_flow_item_type)
5501                                 RTE_FLOW_ITEM_TYPE_END,
5502                 };
5503                 /* Prepare the tag action in prefix subflow. */
5504                 actions_pre[index++] =
5505                         (struct rte_flow_action){
5506                         .type = (enum rte_flow_action_type)
5507                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5508                         .conf = set_tag,
5509                 };
5510         }
5511         memcpy(actions_pre + index, actions + sample_action_pos,
5512                sizeof(struct rte_flow_action));
5513         index += 1;
5514         /* For the modify action after the sample action in E-Switch mirroring,
5515          * Add the extra jump action in prefix subflow and jump into the next
5516          * table, then do the modify action in the new table.
5517          */
5518         if (jump_table) {
5519                 /* Prepare the prefix jump action. */
5520                 append_index++;
5521                 jump_action = (void *)(actions_pre + actions_n + append_index);
5522                 jump_action->group = jump_table;
5523                 actions_pre[index++] =
5524                         (struct rte_flow_action){
5525                         .type = (enum rte_flow_action_type)
5526                                 RTE_FLOW_ACTION_TYPE_JUMP,
5527                         .conf = jump_action,
5528                 };
5529         }
5530         actions_pre[index] = (struct rte_flow_action){
5531                 .type = (enum rte_flow_action_type)
5532                         RTE_FLOW_ACTION_TYPE_END,
5533         };
5534         /* Put the actions after sample into Suffix flow. */
5535         memcpy(actions_sfx, actions + sample_action_pos + 1,
5536                sizeof(struct rte_flow_action) *
5537                (actions_n - sample_action_pos - 1));
5538         return tag_id;
5539 }
5540
5541 /**
5542  * The splitting for metadata feature.
5543  *
5544  * - Q/RSS action on NIC Rx should be split in order to pass by
5545  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5546  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5547  *
5548  * - All the actions on NIC Tx should have a mreg copy action to
5549  *   copy reg_a from WQE to reg_c[0].
5550  *
5551  * @param dev
5552  *   Pointer to Ethernet device.
5553  * @param[in] flow
5554  *   Parent flow structure pointer.
5555  * @param[in] attr
5556  *   Flow rule attributes.
5557  * @param[in] items
5558  *   Pattern specification (list terminated by the END pattern item).
5559  * @param[in] actions
5560  *   Associated actions (list terminated by the END action).
5561  * @param[in] flow_split_info
5562  *   Pointer to flow split info structure.
5563  * @param[out] error
5564  *   Perform verbose error reporting if not NULL.
5565  * @return
5566  *   0 on success, negative value otherwise
5567  */
5568 static int
5569 flow_create_split_metadata(struct rte_eth_dev *dev,
5570                            struct rte_flow *flow,
5571                            const struct rte_flow_attr *attr,
5572                            const struct rte_flow_item items[],
5573                            const struct rte_flow_action actions[],
5574                            struct mlx5_flow_split_info *flow_split_info,
5575                            struct rte_flow_error *error)
5576 {
5577         struct mlx5_priv *priv = dev->data->dev_private;
5578         struct mlx5_dev_config *config = &priv->config;
5579         const struct rte_flow_action *qrss = NULL;
5580         struct rte_flow_action *ext_actions = NULL;
5581         struct mlx5_flow *dev_flow = NULL;
5582         uint32_t qrss_id = 0;
5583         int mtr_sfx = 0;
5584         size_t act_size;
5585         int actions_n;
5586         int encap_idx;
5587         int ret;
5588
5589         /* Check whether extensive metadata feature is engaged. */
5590         if (!config->dv_flow_en ||
5591             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5592             !mlx5_flow_ext_mreg_supported(dev))
5593                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5594                                                actions, flow_split_info, error);
5595         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5596                                                            &encap_idx);
5597         if (qrss) {
5598                 /* Exclude hairpin flows from splitting. */
5599                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5600                         const struct rte_flow_action_queue *queue;
5601
5602                         queue = qrss->conf;
5603                         if (mlx5_rxq_get_type(dev, queue->index) ==
5604                             MLX5_RXQ_TYPE_HAIRPIN)
5605                                 qrss = NULL;
5606                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5607                         const struct rte_flow_action_rss *rss;
5608
5609                         rss = qrss->conf;
5610                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5611                             MLX5_RXQ_TYPE_HAIRPIN)
5612                                 qrss = NULL;
5613                 }
5614         }
5615         if (qrss) {
5616                 /* Check if it is in meter suffix table. */
5617                 mtr_sfx = attr->group == (attr->transfer ?
5618                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5619                           MLX5_FLOW_TABLE_LEVEL_METER);
5620                 /*
5621                  * Q/RSS action on NIC Rx should be split in order to pass by
5622                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5623                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5624                  */
5625                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5626                            sizeof(struct rte_flow_action_set_tag) +
5627                            sizeof(struct rte_flow_action_jump);
5628                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5629                                           SOCKET_ID_ANY);
5630                 if (!ext_actions)
5631                         return rte_flow_error_set(error, ENOMEM,
5632                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5633                                                   NULL, "no memory to split "
5634                                                   "metadata flow");
5635                 /*
5636                  * If we are the suffix flow of meter, tag already exist.
5637                  * Set the tag action to void.
5638                  */
5639                 if (mtr_sfx)
5640                         ext_actions[qrss - actions].type =
5641                                                 RTE_FLOW_ACTION_TYPE_VOID;
5642                 else
5643                         ext_actions[qrss - actions].type =
5644                                                 (enum rte_flow_action_type)
5645                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5646                 /*
5647                  * Create the new actions list with removed Q/RSS action
5648                  * and appended set tag and jump to register copy table
5649                  * (RX_CP_TBL). We should preallocate unique tag ID here
5650                  * in advance, because it is needed for set tag action.
5651                  */
5652                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5653                                                     qrss, actions_n, error);
5654                 if (!mtr_sfx && !qrss_id) {
5655                         ret = -rte_errno;
5656                         goto exit;
5657                 }
5658         } else if (attr->egress && !attr->transfer) {
5659                 /*
5660                  * All the actions on NIC Tx should have a metadata register
5661                  * copy action to copy reg_a from WQE to reg_c[meta]
5662                  */
5663                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5664                            sizeof(struct mlx5_flow_action_copy_mreg);
5665                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5666                                           SOCKET_ID_ANY);
5667                 if (!ext_actions)
5668                         return rte_flow_error_set(error, ENOMEM,
5669                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5670                                                   NULL, "no memory to split "
5671                                                   "metadata flow");
5672                 /* Create the action list appended with copy register. */
5673                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5674                                              actions_n, error, encap_idx);
5675                 if (ret < 0)
5676                         goto exit;
5677         }
5678         /* Add the unmodified original or prefix subflow. */
5679         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5680                                       items, ext_actions ? ext_actions :
5681                                       actions, flow_split_info, error);
5682         if (ret < 0)
5683                 goto exit;
5684         MLX5_ASSERT(dev_flow);
5685         if (qrss) {
5686                 const struct rte_flow_attr q_attr = {
5687                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5688                         .ingress = 1,
5689                 };
5690                 /* Internal PMD action to set register. */
5691                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5692                         .data = qrss_id,
5693                         .id = REG_NON,
5694                 };
5695                 struct rte_flow_item q_items[] = {
5696                         {
5697                                 .type = (enum rte_flow_item_type)
5698                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5699                                 .spec = &q_tag_spec,
5700                                 .last = NULL,
5701                                 .mask = NULL,
5702                         },
5703                         {
5704                                 .type = RTE_FLOW_ITEM_TYPE_END,
5705                         },
5706                 };
5707                 struct rte_flow_action q_actions[] = {
5708                         {
5709                                 .type = qrss->type,
5710                                 .conf = qrss->conf,
5711                         },
5712                         {
5713                                 .type = RTE_FLOW_ACTION_TYPE_END,
5714                         },
5715                 };
5716                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5717
5718                 /*
5719                  * Configure the tag item only if there is no meter subflow.
5720                  * Since tag is already marked in the meter suffix subflow
5721                  * we can just use the meter suffix items as is.
5722                  */
5723                 if (qrss_id) {
5724                         /* Not meter subflow. */
5725                         MLX5_ASSERT(!mtr_sfx);
5726                         /*
5727                          * Put unique id in prefix flow due to it is destroyed
5728                          * after suffix flow and id will be freed after there
5729                          * is no actual flows with this id and identifier
5730                          * reallocation becomes possible (for example, for
5731                          * other flows in other threads).
5732                          */
5733                         dev_flow->handle->split_flow_id = qrss_id;
5734                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5735                                                    error);
5736                         if (ret < 0)
5737                                 goto exit;
5738                         q_tag_spec.id = ret;
5739                 }
5740                 dev_flow = NULL;
5741                 /* Add suffix subflow to execute Q/RSS. */
5742                 flow_split_info->prefix_layers = layers;
5743                 flow_split_info->prefix_mark = 0;
5744                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5745                                               &q_attr, mtr_sfx ? items :
5746                                               q_items, q_actions,
5747                                               flow_split_info, error);
5748                 if (ret < 0)
5749                         goto exit;
5750                 /* qrss ID should be freed if failed. */
5751                 qrss_id = 0;
5752                 MLX5_ASSERT(dev_flow);
5753         }
5754
5755 exit:
5756         /*
5757          * We do not destroy the partially created sub_flows in case of error.
5758          * These ones are included into parent flow list and will be destroyed
5759          * by flow_drv_destroy.
5760          */
5761         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5762                         qrss_id);
5763         mlx5_free(ext_actions);
5764         return ret;
5765 }
5766
5767 /**
5768  * Create meter internal drop flow with the original pattern.
5769  *
5770  * @param dev
5771  *   Pointer to Ethernet device.
5772  * @param[in] flow
5773  *   Parent flow structure pointer.
5774  * @param[in] attr
5775  *   Flow rule attributes.
5776  * @param[in] items
5777  *   Pattern specification (list terminated by the END pattern item).
5778  * @param[in] flow_split_info
5779  *   Pointer to flow split info structure.
5780  * @param[in] fm
5781  *   Pointer to flow meter structure.
5782  * @param[out] error
5783  *   Perform verbose error reporting if not NULL.
5784  * @return
5785  *   0 on success, negative value otherwise
5786  */
5787 static uint32_t
5788 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5789                         struct rte_flow *flow,
5790                         const struct rte_flow_attr *attr,
5791                         const struct rte_flow_item items[],
5792                         struct mlx5_flow_split_info *flow_split_info,
5793                         struct mlx5_flow_meter_info *fm,
5794                         struct rte_flow_error *error)
5795 {
5796         struct mlx5_flow *dev_flow = NULL;
5797         struct rte_flow_attr drop_attr = *attr;
5798         struct rte_flow_action drop_actions[3];
5799         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5800
5801         MLX5_ASSERT(fm->drop_cnt);
5802         drop_actions[0].type =
5803                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5804         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5805         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5806         drop_actions[1].conf = NULL;
5807         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5808         drop_actions[2].conf = NULL;
5809         drop_split_info.external = false;
5810         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5811         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5812         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5813         return flow_create_split_inner(dev, flow, &dev_flow,
5814                                 &drop_attr, items, drop_actions,
5815                                 &drop_split_info, error);
5816 }
5817
5818 /**
5819  * The splitting for meter feature.
5820  *
5821  * - The meter flow will be split to two flows as prefix and
5822  *   suffix flow. The packets make sense only it pass the prefix
5823  *   meter action.
5824  *
5825  * - Reg_C_5 is used for the packet to match betweend prefix and
5826  *   suffix flow.
5827  *
5828  * @param dev
5829  *   Pointer to Ethernet device.
5830  * @param[in] flow
5831  *   Parent flow structure pointer.
5832  * @param[in] attr
5833  *   Flow rule attributes.
5834  * @param[in] items
5835  *   Pattern specification (list terminated by the END pattern item).
5836  * @param[in] actions
5837  *   Associated actions (list terminated by the END action).
5838  * @param[in] flow_split_info
5839  *   Pointer to flow split info structure.
5840  * @param[out] error
5841  *   Perform verbose error reporting if not NULL.
5842  * @return
5843  *   0 on success, negative value otherwise
5844  */
5845 static int
5846 flow_create_split_meter(struct rte_eth_dev *dev,
5847                         struct rte_flow *flow,
5848                         const struct rte_flow_attr *attr,
5849                         const struct rte_flow_item items[],
5850                         const struct rte_flow_action actions[],
5851                         struct mlx5_flow_split_info *flow_split_info,
5852                         struct rte_flow_error *error)
5853 {
5854         struct mlx5_priv *priv = dev->data->dev_private;
5855         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5856         struct rte_flow_action *sfx_actions = NULL;
5857         struct rte_flow_action *pre_actions = NULL;
5858         struct rte_flow_item *sfx_items = NULL;
5859         struct mlx5_flow *dev_flow = NULL;
5860         struct rte_flow_attr sfx_attr = *attr;
5861         struct mlx5_flow_meter_info *fm = NULL;
5862         uint8_t skip_scale_restore;
5863         bool has_mtr = false;
5864         bool has_modify = false;
5865         bool set_mtr_reg = true;
5866         bool is_mtr_hierarchy = false;
5867         uint32_t meter_id = 0;
5868         uint32_t mtr_idx = 0;
5869         uint32_t mtr_flow_id = 0;
5870         size_t act_size;
5871         size_t item_size;
5872         int actions_n = 0;
5873         int ret = 0;
5874
5875         if (priv->mtr_en)
5876                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5877                                                     &has_modify, &meter_id);
5878         if (has_mtr) {
5879                 if (flow->meter) {
5880                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5881                         if (!fm)
5882                                 return rte_flow_error_set(error, EINVAL,
5883                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5884                                                 NULL, "Meter not found.");
5885                 } else {
5886                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5887                         if (!fm)
5888                                 return rte_flow_error_set(error, EINVAL,
5889                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5890                                                 NULL, "Meter not found.");
5891                         ret = mlx5_flow_meter_attach(priv, fm,
5892                                                      &sfx_attr, error);
5893                         if (ret)
5894                                 return -rte_errno;
5895                         flow->meter = mtr_idx;
5896                 }
5897                 MLX5_ASSERT(wks);
5898                 wks->fm = fm;
5899                 if (!fm->def_policy) {
5900                         wks->policy = mlx5_flow_meter_policy_find(dev,
5901                                                                   fm->policy_id,
5902                                                                   NULL);
5903                         MLX5_ASSERT(wks->policy);
5904                         if (wks->policy->is_hierarchy) {
5905                                 wks->final_policy =
5906                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
5907                                                                 wks->policy);
5908                                 if (!wks->final_policy)
5909                                         return rte_flow_error_set(error,
5910                                         EINVAL,
5911                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5912                                 "Failed to find terminal policy of hierarchy.");
5913                                 is_mtr_hierarchy = true;
5914                         }
5915                 }
5916                 /*
5917                  * If it isn't default-policy Meter, and
5918                  * 1. There's no action in flow to change
5919                  *    packet (modify/encap/decap etc.), OR
5920                  * 2. No drop count needed for this meter.
5921                  * 3. It's not meter hierarchy.
5922                  * Then no need to use regC to save meter id anymore.
5923                  */
5924                 if (!fm->def_policy && !is_mtr_hierarchy &&
5925                     (!has_modify || !fm->drop_cnt))
5926                         set_mtr_reg = false;
5927                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5928                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5929                            sizeof(struct mlx5_rte_flow_action_set_tag);
5930                 /* Suffix items: tag, vlan, port id, end. */
5931 #define METER_SUFFIX_ITEM 4
5932                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5933                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5934                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5935                                           0, SOCKET_ID_ANY);
5936                 if (!sfx_actions)
5937                         return rte_flow_error_set(error, ENOMEM,
5938                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5939                                                   NULL, "no memory to split "
5940                                                   "meter flow");
5941                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5942                              act_size);
5943                 /* There's no suffix flow for meter of non-default policy. */
5944                 if (!fm->def_policy)
5945                         pre_actions = sfx_actions + 1;
5946                 else
5947                         pre_actions = sfx_actions + actions_n;
5948                 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
5949                                             items, sfx_items, actions,
5950                                             sfx_actions, pre_actions,
5951                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5952                                             error);
5953                 if (ret) {
5954                         ret = -rte_errno;
5955                         goto exit;
5956                 }
5957                 /* Add the prefix subflow. */
5958                 flow_split_info->prefix_mark = 0;
5959                 skip_scale_restore = flow_split_info->skip_scale;
5960                 flow_split_info->skip_scale |=
5961                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5962                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5963                                               attr, items, pre_actions,
5964                                               flow_split_info, error);
5965                 flow_split_info->skip_scale = skip_scale_restore;
5966                 if (ret) {
5967                         if (mtr_flow_id)
5968                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5969                         ret = -rte_errno;
5970                         goto exit;
5971                 }
5972                 if (mtr_flow_id) {
5973                         dev_flow->handle->split_flow_id = mtr_flow_id;
5974                         dev_flow->handle->is_meter_flow_id = 1;
5975                 }
5976                 if (!fm->def_policy) {
5977                         if (!set_mtr_reg && fm->drop_cnt)
5978                                 ret =
5979                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5980                                                         &sfx_attr, items,
5981                                                         flow_split_info,
5982                                                         fm, error);
5983                         goto exit;
5984                 }
5985                 /* Setting the sfx group atrr. */
5986                 sfx_attr.group = sfx_attr.transfer ?
5987                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5988                                  MLX5_FLOW_TABLE_LEVEL_METER;
5989                 flow_split_info->prefix_layers =
5990                                 flow_get_prefix_layer_flags(dev_flow);
5991                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5992                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5993         }
5994         /* Add the prefix subflow. */
5995         ret = flow_create_split_metadata(dev, flow,
5996                                          &sfx_attr, sfx_items ?
5997                                          sfx_items : items,
5998                                          sfx_actions ? sfx_actions : actions,
5999                                          flow_split_info, error);
6000 exit:
6001         if (sfx_actions)
6002                 mlx5_free(sfx_actions);
6003         return ret;
6004 }
6005
6006 /**
6007  * The splitting for sample feature.
6008  *
6009  * Once Sample action is detected in the action list, the flow actions should
6010  * be split into prefix sub flow and suffix sub flow.
6011  *
6012  * The original items remain in the prefix sub flow, all actions preceding the
6013  * sample action and the sample action itself will be copied to the prefix
6014  * sub flow, the actions following the sample action will be copied to the
6015  * suffix sub flow, Queue action always be located in the suffix sub flow.
6016  *
6017  * In order to make the packet from prefix sub flow matches with suffix sub
6018  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6019  * flow uses tag item with the unique flow id.
6020  *
6021  * @param dev
6022  *   Pointer to Ethernet device.
6023  * @param[in] flow
6024  *   Parent flow structure pointer.
6025  * @param[in] attr
6026  *   Flow rule attributes.
6027  * @param[in] items
6028  *   Pattern specification (list terminated by the END pattern item).
6029  * @param[in] actions
6030  *   Associated actions (list terminated by the END action).
6031  * @param[in] flow_split_info
6032  *   Pointer to flow split info structure.
6033  * @param[out] error
6034  *   Perform verbose error reporting if not NULL.
6035  * @return
6036  *   0 on success, negative value otherwise
6037  */
6038 static int
6039 flow_create_split_sample(struct rte_eth_dev *dev,
6040                          struct rte_flow *flow,
6041                          const struct rte_flow_attr *attr,
6042                          const struct rte_flow_item items[],
6043                          const struct rte_flow_action actions[],
6044                          struct mlx5_flow_split_info *flow_split_info,
6045                          struct rte_flow_error *error)
6046 {
6047         struct mlx5_priv *priv = dev->data->dev_private;
6048         struct rte_flow_action *sfx_actions = NULL;
6049         struct rte_flow_action *pre_actions = NULL;
6050         struct rte_flow_item *sfx_items = NULL;
6051         struct mlx5_flow *dev_flow = NULL;
6052         struct rte_flow_attr sfx_attr = *attr;
6053 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6054         struct mlx5_flow_dv_sample_resource *sample_res;
6055         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6056         struct mlx5_flow_tbl_resource *sfx_tbl;
6057 #endif
6058         size_t act_size;
6059         size_t item_size;
6060         uint32_t fdb_tx = 0;
6061         int32_t tag_id = 0;
6062         int actions_n = 0;
6063         int sample_action_pos;
6064         int qrss_action_pos;
6065         int add_tag = 0;
6066         int modify_after_mirror = 0;
6067         uint16_t jump_table = 0;
6068         const uint32_t next_ft_step = 1;
6069         int ret = 0;
6070
6071         if (priv->sampler_en)
6072                 actions_n = flow_check_match_action(actions, attr,
6073                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
6074                                         &sample_action_pos, &qrss_action_pos,
6075                                         &modify_after_mirror);
6076         if (actions_n) {
6077                 /* The prefix actions must includes sample, tag, end. */
6078                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6079                            + sizeof(struct mlx5_rte_flow_action_set_tag);
6080                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6081                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6082                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6083                                           item_size), 0, SOCKET_ID_ANY);
6084                 if (!sfx_actions)
6085                         return rte_flow_error_set(error, ENOMEM,
6086                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6087                                                   NULL, "no memory to split "
6088                                                   "sample flow");
6089                 /* The representor_id is UINT16_MAX for uplink. */
6090                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6091                 /*
6092                  * When reg_c_preserve is set, metadata registers Cx preserve
6093                  * their value even through packet duplication.
6094                  */
6095                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
6096                 if (add_tag)
6097                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6098                                         + act_size);
6099                 if (modify_after_mirror)
6100                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6101                                      next_ft_step;
6102                 pre_actions = sfx_actions + actions_n;
6103                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
6104                                                 actions, sfx_actions,
6105                                                 pre_actions, actions_n,
6106                                                 sample_action_pos,
6107                                                 qrss_action_pos, jump_table,
6108                                                 error);
6109                 if (tag_id < 0 || (add_tag && !tag_id)) {
6110                         ret = -rte_errno;
6111                         goto exit;
6112                 }
6113                 if (modify_after_mirror)
6114                         flow_split_info->skip_scale =
6115                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6116                 /* Add the prefix subflow. */
6117                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6118                                               items, pre_actions,
6119                                               flow_split_info, error);
6120                 if (ret) {
6121                         ret = -rte_errno;
6122                         goto exit;
6123                 }
6124                 dev_flow->handle->split_flow_id = tag_id;
6125 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6126                 if (!modify_after_mirror) {
6127                         /* Set the sfx group attr. */
6128                         sample_res = (struct mlx5_flow_dv_sample_resource *)
6129                                                 dev_flow->dv.sample_res;
6130                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
6131                                                 sample_res->normal_path_tbl;
6132                         sfx_tbl_data = container_of(sfx_tbl,
6133                                                 struct mlx5_flow_tbl_data_entry,
6134                                                 tbl);
6135                         sfx_attr.group = sfx_attr.transfer ?
6136                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6137                 } else {
6138                         MLX5_ASSERT(attr->transfer);
6139                         sfx_attr.group = jump_table;
6140                 }
6141                 flow_split_info->prefix_layers =
6142                                 flow_get_prefix_layer_flags(dev_flow);
6143                 flow_split_info->prefix_mark = dev_flow->handle->mark;
6144                 /* Suffix group level already be scaled with factor, set
6145                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6146                  * again in translation.
6147                  */
6148                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6149 #endif
6150         }
6151         /* Add the suffix subflow. */
6152         ret = flow_create_split_meter(dev, flow, &sfx_attr,
6153                                       sfx_items ? sfx_items : items,
6154                                       sfx_actions ? sfx_actions : actions,
6155                                       flow_split_info, error);
6156 exit:
6157         if (sfx_actions)
6158                 mlx5_free(sfx_actions);
6159         return ret;
6160 }
6161
6162 /**
6163  * Split the flow to subflow set. The splitters might be linked
6164  * in the chain, like this:
6165  * flow_create_split_outer() calls:
6166  *   flow_create_split_meter() calls:
6167  *     flow_create_split_metadata(meter_subflow_0) calls:
6168  *       flow_create_split_inner(metadata_subflow_0)
6169  *       flow_create_split_inner(metadata_subflow_1)
6170  *       flow_create_split_inner(metadata_subflow_2)
6171  *     flow_create_split_metadata(meter_subflow_1) calls:
6172  *       flow_create_split_inner(metadata_subflow_0)
6173  *       flow_create_split_inner(metadata_subflow_1)
6174  *       flow_create_split_inner(metadata_subflow_2)
6175  *
6176  * This provide flexible way to add new levels of flow splitting.
6177  * The all of successfully created subflows are included to the
6178  * parent flow dev_flow list.
6179  *
6180  * @param dev
6181  *   Pointer to Ethernet device.
6182  * @param[in] flow
6183  *   Parent flow structure pointer.
6184  * @param[in] attr
6185  *   Flow rule attributes.
6186  * @param[in] items
6187  *   Pattern specification (list terminated by the END pattern item).
6188  * @param[in] actions
6189  *   Associated actions (list terminated by the END action).
6190  * @param[in] flow_split_info
6191  *   Pointer to flow split info structure.
6192  * @param[out] error
6193  *   Perform verbose error reporting if not NULL.
6194  * @return
6195  *   0 on success, negative value otherwise
6196  */
6197 static int
6198 flow_create_split_outer(struct rte_eth_dev *dev,
6199                         struct rte_flow *flow,
6200                         const struct rte_flow_attr *attr,
6201                         const struct rte_flow_item items[],
6202                         const struct rte_flow_action actions[],
6203                         struct mlx5_flow_split_info *flow_split_info,
6204                         struct rte_flow_error *error)
6205 {
6206         int ret;
6207
6208         ret = flow_create_split_sample(dev, flow, attr, items,
6209                                        actions, flow_split_info, error);
6210         MLX5_ASSERT(ret <= 0);
6211         return ret;
6212 }
6213
6214 static inline struct mlx5_flow_tunnel *
6215 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6216 {
6217         struct mlx5_flow_tunnel *tunnel;
6218
6219 #pragma GCC diagnostic push
6220 #pragma GCC diagnostic ignored "-Wcast-qual"
6221         tunnel = (typeof(tunnel))flow->tunnel;
6222 #pragma GCC diagnostic pop
6223
6224         return tunnel;
6225 }
6226
6227 /**
6228  * Adjust flow RSS workspace if needed.
6229  *
6230  * @param wks
6231  *   Pointer to thread flow work space.
6232  * @param rss_desc
6233  *   Pointer to RSS descriptor.
6234  * @param[in] nrssq_num
6235  *   New RSS queue number.
6236  *
6237  * @return
6238  *   0 on success, -1 otherwise and rte_errno is set.
6239  */
6240 static int
6241 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6242                           struct mlx5_flow_rss_desc *rss_desc,
6243                           uint32_t nrssq_num)
6244 {
6245         if (likely(nrssq_num <= wks->rssq_num))
6246                 return 0;
6247         rss_desc->queue = realloc(rss_desc->queue,
6248                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6249         if (!rss_desc->queue) {
6250                 rte_errno = ENOMEM;
6251                 return -1;
6252         }
6253         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6254         return 0;
6255 }
6256
6257 /**
6258  * Create a flow and add it to @p list.
6259  *
6260  * @param dev
6261  *   Pointer to Ethernet device.
6262  * @param list
6263  *   Pointer to a TAILQ flow list. If this parameter NULL,
6264  *   no list insertion occurred, flow is just created,
6265  *   this is caller's responsibility to track the
6266  *   created flow.
6267  * @param[in] attr
6268  *   Flow rule attributes.
6269  * @param[in] items
6270  *   Pattern specification (list terminated by the END pattern item).
6271  * @param[in] actions
6272  *   Associated actions (list terminated by the END action).
6273  * @param[in] external
6274  *   This flow rule is created by request external to PMD.
6275  * @param[out] error
6276  *   Perform verbose error reporting if not NULL.
6277  *
6278  * @return
6279  *   A flow index on success, 0 otherwise and rte_errno is set.
6280  */
6281 static uint32_t
6282 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6283                  const struct rte_flow_attr *attr,
6284                  const struct rte_flow_item items[],
6285                  const struct rte_flow_action original_actions[],
6286                  bool external, struct rte_flow_error *error)
6287 {
6288         struct mlx5_priv *priv = dev->data->dev_private;
6289         struct rte_flow *flow = NULL;
6290         struct mlx5_flow *dev_flow;
6291         const struct rte_flow_action_rss *rss = NULL;
6292         struct mlx5_translated_action_handle
6293                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6294         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6295         union {
6296                 struct mlx5_flow_expand_rss buf;
6297                 uint8_t buffer[4096];
6298         } expand_buffer;
6299         union {
6300                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6301                 uint8_t buffer[2048];
6302         } actions_rx;
6303         union {
6304                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6305                 uint8_t buffer[2048];
6306         } actions_hairpin_tx;
6307         union {
6308                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6309                 uint8_t buffer[2048];
6310         } items_tx;
6311         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6312         struct mlx5_flow_rss_desc *rss_desc;
6313         const struct rte_flow_action *p_actions_rx;
6314         uint32_t i;
6315         uint32_t idx = 0;
6316         int hairpin_flow;
6317         struct rte_flow_attr attr_tx = { .priority = 0 };
6318         const struct rte_flow_action *actions;
6319         struct rte_flow_action *translated_actions = NULL;
6320         struct mlx5_flow_tunnel *tunnel;
6321         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6322         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6323         struct mlx5_flow_split_info flow_split_info = {
6324                 .external = !!external,
6325                 .skip_scale = 0,
6326                 .flow_idx = 0,
6327                 .prefix_mark = 0,
6328                 .prefix_layers = 0,
6329                 .table_id = 0
6330         };
6331         int ret;
6332
6333         MLX5_ASSERT(wks);
6334         rss_desc = &wks->rss_desc;
6335         ret = flow_action_handles_translate(dev, original_actions,
6336                                             indir_actions,
6337                                             &indir_actions_n,
6338                                             &translated_actions, error);
6339         if (ret < 0) {
6340                 MLX5_ASSERT(translated_actions == NULL);
6341                 return 0;
6342         }
6343         actions = translated_actions ? translated_actions : original_actions;
6344         p_actions_rx = actions;
6345         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6346         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6347                                 external, hairpin_flow, error);
6348         if (ret < 0)
6349                 goto error_before_hairpin_split;
6350         flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6351         if (!flow) {
6352                 rte_errno = ENOMEM;
6353                 goto error_before_hairpin_split;
6354         }
6355         if (hairpin_flow > 0) {
6356                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6357                         rte_errno = EINVAL;
6358                         goto error_before_hairpin_split;
6359                 }
6360                 flow_hairpin_split(dev, actions, actions_rx.actions,
6361                                    actions_hairpin_tx.actions, items_tx.items,
6362                                    idx);
6363                 p_actions_rx = actions_rx.actions;
6364         }
6365         flow_split_info.flow_idx = idx;
6366         flow->drv_type = flow_get_drv_type(dev, attr);
6367         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6368                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6369         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6370         /* RSS Action only works on NIC RX domain */
6371         if (attr->ingress && !attr->transfer)
6372                 rss = flow_get_rss_action(dev, p_actions_rx);
6373         if (rss) {
6374                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6375                         return 0;
6376                 /*
6377                  * The following information is required by
6378                  * mlx5_flow_hashfields_adjust() in advance.
6379                  */
6380                 rss_desc->level = rss->level;
6381                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6382                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6383         }
6384         flow->dev_handles = 0;
6385         if (rss && rss->types) {
6386                 unsigned int graph_root;
6387
6388                 graph_root = find_graph_root(rss->level);
6389                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6390                                            items, rss->types,
6391                                            mlx5_support_expansion, graph_root);
6392                 MLX5_ASSERT(ret > 0 &&
6393                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6394                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6395                         for (i = 0; i < buf->entries; ++i)
6396                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6397                 }
6398         } else {
6399                 buf->entries = 1;
6400                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6401         }
6402         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6403                                                       indir_actions_n);
6404         for (i = 0; i < buf->entries; ++i) {
6405                 /* Initialize flow split data. */
6406                 flow_split_info.prefix_layers = 0;
6407                 flow_split_info.prefix_mark = 0;
6408                 flow_split_info.skip_scale = 0;
6409                 /*
6410                  * The splitter may create multiple dev_flows,
6411                  * depending on configuration. In the simplest
6412                  * case it just creates unmodified original flow.
6413                  */
6414                 ret = flow_create_split_outer(dev, flow, attr,
6415                                               buf->entry[i].pattern,
6416                                               p_actions_rx, &flow_split_info,
6417                                               error);
6418                 if (ret < 0)
6419                         goto error;
6420                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6421                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6422                                                            p_actions_rx,
6423                                                            idx,
6424                                                            wks->flows[0].tunnel,
6425                                                            &default_miss_ctx,
6426                                                            error);
6427                         if (ret < 0) {
6428                                 mlx5_free(default_miss_ctx.queue);
6429                                 goto error;
6430                         }
6431                 }
6432         }
6433         /* Create the tx flow. */
6434         if (hairpin_flow) {
6435                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6436                 attr_tx.ingress = 0;
6437                 attr_tx.egress = 1;
6438                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6439                                          actions_hairpin_tx.actions,
6440                                          idx, error);
6441                 if (!dev_flow)
6442                         goto error;
6443                 dev_flow->flow = flow;
6444                 dev_flow->external = 0;
6445                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6446                               dev_flow->handle, next);
6447                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6448                                          items_tx.items,
6449                                          actions_hairpin_tx.actions, error);
6450                 if (ret < 0)
6451                         goto error;
6452         }
6453         /*
6454          * Update the metadata register copy table. If extensive
6455          * metadata feature is enabled and registers are supported
6456          * we might create the extra rte_flow for each unique
6457          * MARK/FLAG action ID.
6458          *
6459          * The table is updated for ingress Flows only, because
6460          * the egress Flows belong to the different device and
6461          * copy table should be updated in peer NIC Rx domain.
6462          */
6463         if (attr->ingress &&
6464             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6465                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6466                 if (ret)
6467                         goto error;
6468         }
6469         /*
6470          * If the flow is external (from application) OR device is started,
6471          * OR mreg discover, then apply immediately.
6472          */
6473         if (external || dev->data->dev_started ||
6474             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6475              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6476                 ret = flow_drv_apply(dev, flow, error);
6477                 if (ret < 0)
6478                         goto error;
6479         }
6480         flow->type = type;
6481         flow_rxq_flags_set(dev, flow);
6482         rte_free(translated_actions);
6483         tunnel = flow_tunnel_from_rule(wks->flows);
6484         if (tunnel) {
6485                 flow->tunnel = 1;
6486                 flow->tunnel_id = tunnel->tunnel_id;
6487                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6488                 mlx5_free(default_miss_ctx.queue);
6489         }
6490         mlx5_flow_pop_thread_workspace();
6491         return idx;
6492 error:
6493         MLX5_ASSERT(flow);
6494         ret = rte_errno; /* Save rte_errno before cleanup. */
6495         flow_mreg_del_copy_action(dev, flow);
6496         flow_drv_destroy(dev, flow);
6497         if (rss_desc->shared_rss)
6498                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6499                         mlx5_ipool_get
6500                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6501                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6502         mlx5_ipool_free(priv->flows[type], idx);
6503         rte_errno = ret; /* Restore rte_errno. */
6504         ret = rte_errno;
6505         rte_errno = ret;
6506         mlx5_flow_pop_thread_workspace();
6507 error_before_hairpin_split:
6508         rte_free(translated_actions);
6509         return 0;
6510 }
6511
6512 /**
6513  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6514  * incoming packets to table 1.
6515  *
6516  * Other flow rules, requested for group n, will be created in
6517  * e-switch table n+1.
6518  * Jump action to e-switch group n will be created to group n+1.
6519  *
6520  * Used when working in switchdev mode, to utilise advantages of table 1
6521  * and above.
6522  *
6523  * @param dev
6524  *   Pointer to Ethernet device.
6525  *
6526  * @return
6527  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6528  */
6529 struct rte_flow *
6530 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6531 {
6532         const struct rte_flow_attr attr = {
6533                 .group = 0,
6534                 .priority = 0,
6535                 .ingress = 1,
6536                 .egress = 0,
6537                 .transfer = 1,
6538         };
6539         const struct rte_flow_item pattern = {
6540                 .type = RTE_FLOW_ITEM_TYPE_END,
6541         };
6542         struct rte_flow_action_jump jump = {
6543                 .group = 1,
6544         };
6545         const struct rte_flow_action actions[] = {
6546                 {
6547                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6548                         .conf = &jump,
6549                 },
6550                 {
6551                         .type = RTE_FLOW_ACTION_TYPE_END,
6552                 },
6553         };
6554         struct rte_flow_error error;
6555
6556         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6557                                                    &attr, &pattern,
6558                                                    actions, false, &error);
6559 }
6560
6561 /**
6562  * Validate a flow supported by the NIC.
6563  *
6564  * @see rte_flow_validate()
6565  * @see rte_flow_ops
6566  */
6567 int
6568 mlx5_flow_validate(struct rte_eth_dev *dev,
6569                    const struct rte_flow_attr *attr,
6570                    const struct rte_flow_item items[],
6571                    const struct rte_flow_action original_actions[],
6572                    struct rte_flow_error *error)
6573 {
6574         int hairpin_flow;
6575         struct mlx5_translated_action_handle
6576                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6577         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6578         const struct rte_flow_action *actions;
6579         struct rte_flow_action *translated_actions = NULL;
6580         int ret = flow_action_handles_translate(dev, original_actions,
6581                                                 indir_actions,
6582                                                 &indir_actions_n,
6583                                                 &translated_actions, error);
6584
6585         if (ret)
6586                 return ret;
6587         actions = translated_actions ? translated_actions : original_actions;
6588         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6589         ret = flow_drv_validate(dev, attr, items, actions,
6590                                 true, hairpin_flow, error);
6591         rte_free(translated_actions);
6592         return ret;
6593 }
6594
6595 /**
6596  * Create a flow.
6597  *
6598  * @see rte_flow_create()
6599  * @see rte_flow_ops
6600  */
6601 struct rte_flow *
6602 mlx5_flow_create(struct rte_eth_dev *dev,
6603                  const struct rte_flow_attr *attr,
6604                  const struct rte_flow_item items[],
6605                  const struct rte_flow_action actions[],
6606                  struct rte_flow_error *error)
6607 {
6608         /*
6609          * If the device is not started yet, it is not allowed to created a
6610          * flow from application. PMD default flows and traffic control flows
6611          * are not affected.
6612          */
6613         if (unlikely(!dev->data->dev_started)) {
6614                 DRV_LOG(DEBUG, "port %u is not started when "
6615                         "inserting a flow", dev->data->port_id);
6616                 rte_flow_error_set(error, ENODEV,
6617                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6618                                    NULL,
6619                                    "port not started");
6620                 return NULL;
6621         }
6622
6623         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
6624                                                    attr, items, actions,
6625                                                    true, error);
6626 }
6627
6628 /**
6629  * Destroy a flow in a list.
6630  *
6631  * @param dev
6632  *   Pointer to Ethernet device.
6633  * @param[in] flow_idx
6634  *   Index of flow to destroy.
6635  */
6636 static void
6637 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6638                   uint32_t flow_idx)
6639 {
6640         struct mlx5_priv *priv = dev->data->dev_private;
6641         struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
6642
6643         if (!flow)
6644                 return;
6645         MLX5_ASSERT(flow->type == type);
6646         /*
6647          * Update RX queue flags only if port is started, otherwise it is
6648          * already clean.
6649          */
6650         if (dev->data->dev_started)
6651                 flow_rxq_flags_trim(dev, flow);
6652         flow_drv_destroy(dev, flow);
6653         if (flow->tunnel) {
6654                 struct mlx5_flow_tunnel *tunnel;
6655
6656                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6657                 RTE_VERIFY(tunnel);
6658                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6659                         mlx5_flow_tunnel_free(dev, tunnel);
6660         }
6661         flow_mreg_del_copy_action(dev, flow);
6662         mlx5_ipool_free(priv->flows[type], flow_idx);
6663 }
6664
6665 /**
6666  * Destroy all flows.
6667  *
6668  * @param dev
6669  *   Pointer to Ethernet device.
6670  * @param type
6671  *   Flow type to be flushed.
6672  * @param active
6673  *   If flushing is called avtively.
6674  */
6675 void
6676 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6677                      bool active)
6678 {
6679         struct mlx5_priv *priv = dev->data->dev_private;
6680         uint32_t num_flushed = 0, fidx = 1;
6681         struct rte_flow *flow;
6682
6683         MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
6684                 flow_list_destroy(dev, type, fidx);
6685                 num_flushed++;
6686         }
6687         if (active) {
6688                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6689                         dev->data->port_id, num_flushed);
6690         }
6691 }
6692
6693 /**
6694  * Stop all default actions for flows.
6695  *
6696  * @param dev
6697  *   Pointer to Ethernet device.
6698  */
6699 void
6700 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6701 {
6702         flow_mreg_del_default_copy_action(dev);
6703         flow_rxq_flags_clear(dev);
6704 }
6705
6706 /**
6707  * Start all default actions for flows.
6708  *
6709  * @param dev
6710  *   Pointer to Ethernet device.
6711  * @return
6712  *   0 on success, a negative errno value otherwise and rte_errno is set.
6713  */
6714 int
6715 mlx5_flow_start_default(struct rte_eth_dev *dev)
6716 {
6717         struct rte_flow_error error;
6718
6719         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6720         return flow_mreg_add_default_copy_action(dev, &error);
6721 }
6722
6723 /**
6724  * Release key of thread specific flow workspace data.
6725  */
6726 void
6727 flow_release_workspace(void *data)
6728 {
6729         struct mlx5_flow_workspace *wks = data;
6730         struct mlx5_flow_workspace *next;
6731
6732         while (wks) {
6733                 next = wks->next;
6734                 free(wks->rss_desc.queue);
6735                 free(wks);
6736                 wks = next;
6737         }
6738 }
6739
6740 /**
6741  * Get thread specific current flow workspace.
6742  *
6743  * @return pointer to thread specific flow workspace data, NULL on error.
6744  */
6745 struct mlx5_flow_workspace*
6746 mlx5_flow_get_thread_workspace(void)
6747 {
6748         struct mlx5_flow_workspace *data;
6749
6750         data = mlx5_flow_os_get_specific_workspace();
6751         MLX5_ASSERT(data && data->inuse);
6752         if (!data || !data->inuse)
6753                 DRV_LOG(ERR, "flow workspace not initialized.");
6754         return data;
6755 }
6756
6757 /**
6758  * Allocate and init new flow workspace.
6759  *
6760  * @return pointer to flow workspace data, NULL on error.
6761  */
6762 static struct mlx5_flow_workspace*
6763 flow_alloc_thread_workspace(void)
6764 {
6765         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6766
6767         if (!data) {
6768                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6769                         "memory.");
6770                 return NULL;
6771         }
6772         data->rss_desc.queue = calloc(1,
6773                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6774         if (!data->rss_desc.queue)
6775                 goto err;
6776         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6777         return data;
6778 err:
6779         if (data->rss_desc.queue)
6780                 free(data->rss_desc.queue);
6781         free(data);
6782         return NULL;
6783 }
6784
6785 /**
6786  * Get new thread specific flow workspace.
6787  *
6788  * If current workspace inuse, create new one and set as current.
6789  *
6790  * @return pointer to thread specific flow workspace data, NULL on error.
6791  */
6792 static struct mlx5_flow_workspace*
6793 mlx5_flow_push_thread_workspace(void)
6794 {
6795         struct mlx5_flow_workspace *curr;
6796         struct mlx5_flow_workspace *data;
6797
6798         curr = mlx5_flow_os_get_specific_workspace();
6799         if (!curr) {
6800                 data = flow_alloc_thread_workspace();
6801                 if (!data)
6802                         return NULL;
6803         } else if (!curr->inuse) {
6804                 data = curr;
6805         } else if (curr->next) {
6806                 data = curr->next;
6807         } else {
6808                 data = flow_alloc_thread_workspace();
6809                 if (!data)
6810                         return NULL;
6811                 curr->next = data;
6812                 data->prev = curr;
6813         }
6814         data->inuse = 1;
6815         data->flow_idx = 0;
6816         /* Set as current workspace */
6817         if (mlx5_flow_os_set_specific_workspace(data))
6818                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6819         return data;
6820 }
6821
6822 /**
6823  * Close current thread specific flow workspace.
6824  *
6825  * If previous workspace available, set it as current.
6826  *
6827  * @return pointer to thread specific flow workspace data, NULL on error.
6828  */
6829 static void
6830 mlx5_flow_pop_thread_workspace(void)
6831 {
6832         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6833
6834         if (!data)
6835                 return;
6836         if (!data->inuse) {
6837                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6838                 return;
6839         }
6840         data->inuse = 0;
6841         if (!data->prev)
6842                 return;
6843         if (mlx5_flow_os_set_specific_workspace(data->prev))
6844                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6845 }
6846
6847 /**
6848  * Verify the flow list is empty
6849  *
6850  * @param dev
6851  *  Pointer to Ethernet device.
6852  *
6853  * @return the number of flows not released.
6854  */
6855 int
6856 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
6857 {
6858         struct mlx5_priv *priv = dev->data->dev_private;
6859         struct rte_flow *flow;
6860         uint32_t idx = 0;
6861         int ret = 0, i;
6862
6863         for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
6864                 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
6865                         DRV_LOG(DEBUG, "port %u flow %p still referenced",
6866                                 dev->data->port_id, (void *)flow);
6867                         ret++;
6868                 }
6869         }
6870         return ret;
6871 }
6872
6873 /**
6874  * Enable default hairpin egress flow.
6875  *
6876  * @param dev
6877  *   Pointer to Ethernet device.
6878  * @param queue
6879  *   The queue index.
6880  *
6881  * @return
6882  *   0 on success, a negative errno value otherwise and rte_errno is set.
6883  */
6884 int
6885 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6886                             uint32_t queue)
6887 {
6888         const struct rte_flow_attr attr = {
6889                 .egress = 1,
6890                 .priority = 0,
6891         };
6892         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6893                 .queue = queue,
6894         };
6895         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6896                 .queue = UINT32_MAX,
6897         };
6898         struct rte_flow_item items[] = {
6899                 {
6900                         .type = (enum rte_flow_item_type)
6901                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6902                         .spec = &queue_spec,
6903                         .last = NULL,
6904                         .mask = &queue_mask,
6905                 },
6906                 {
6907                         .type = RTE_FLOW_ITEM_TYPE_END,
6908                 },
6909         };
6910         struct rte_flow_action_jump jump = {
6911                 .group = MLX5_HAIRPIN_TX_TABLE,
6912         };
6913         struct rte_flow_action actions[2];
6914         uint32_t flow_idx;
6915         struct rte_flow_error error;
6916
6917         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6918         actions[0].conf = &jump;
6919         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6920         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6921                                     &attr, items, actions, false, &error);
6922         if (!flow_idx) {
6923                 DRV_LOG(DEBUG,
6924                         "Failed to create ctrl flow: rte_errno(%d),"
6925                         " type(%d), message(%s)",
6926                         rte_errno, error.type,
6927                         error.message ? error.message : " (no stated reason)");
6928                 return -rte_errno;
6929         }
6930         return 0;
6931 }
6932
6933 /**
6934  * Enable a control flow configured from the control plane.
6935  *
6936  * @param dev
6937  *   Pointer to Ethernet device.
6938  * @param eth_spec
6939  *   An Ethernet flow spec to apply.
6940  * @param eth_mask
6941  *   An Ethernet flow mask to apply.
6942  * @param vlan_spec
6943  *   A VLAN flow spec to apply.
6944  * @param vlan_mask
6945  *   A VLAN flow mask to apply.
6946  *
6947  * @return
6948  *   0 on success, a negative errno value otherwise and rte_errno is set.
6949  */
6950 int
6951 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6952                     struct rte_flow_item_eth *eth_spec,
6953                     struct rte_flow_item_eth *eth_mask,
6954                     struct rte_flow_item_vlan *vlan_spec,
6955                     struct rte_flow_item_vlan *vlan_mask)
6956 {
6957         struct mlx5_priv *priv = dev->data->dev_private;
6958         const struct rte_flow_attr attr = {
6959                 .ingress = 1,
6960                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6961         };
6962         struct rte_flow_item items[] = {
6963                 {
6964                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6965                         .spec = eth_spec,
6966                         .last = NULL,
6967                         .mask = eth_mask,
6968                 },
6969                 {
6970                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6971                                               RTE_FLOW_ITEM_TYPE_END,
6972                         .spec = vlan_spec,
6973                         .last = NULL,
6974                         .mask = vlan_mask,
6975                 },
6976                 {
6977                         .type = RTE_FLOW_ITEM_TYPE_END,
6978                 },
6979         };
6980         uint16_t queue[priv->reta_idx_n];
6981         struct rte_flow_action_rss action_rss = {
6982                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6983                 .level = 0,
6984                 .types = priv->rss_conf.rss_hf,
6985                 .key_len = priv->rss_conf.rss_key_len,
6986                 .queue_num = priv->reta_idx_n,
6987                 .key = priv->rss_conf.rss_key,
6988                 .queue = queue,
6989         };
6990         struct rte_flow_action actions[] = {
6991                 {
6992                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6993                         .conf = &action_rss,
6994                 },
6995                 {
6996                         .type = RTE_FLOW_ACTION_TYPE_END,
6997                 },
6998         };
6999         uint32_t flow_idx;
7000         struct rte_flow_error error;
7001         unsigned int i;
7002
7003         if (!priv->reta_idx_n || !priv->rxqs_n) {
7004                 return 0;
7005         }
7006         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7007                 action_rss.types = 0;
7008         for (i = 0; i != priv->reta_idx_n; ++i)
7009                 queue[i] = (*priv->reta_idx)[i];
7010         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7011                                     &attr, items, actions, false, &error);
7012         if (!flow_idx)
7013                 return -rte_errno;
7014         return 0;
7015 }
7016
7017 /**
7018  * Enable a flow control configured from the control plane.
7019  *
7020  * @param dev
7021  *   Pointer to Ethernet device.
7022  * @param eth_spec
7023  *   An Ethernet flow spec to apply.
7024  * @param eth_mask
7025  *   An Ethernet flow mask to apply.
7026  *
7027  * @return
7028  *   0 on success, a negative errno value otherwise and rte_errno is set.
7029  */
7030 int
7031 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7032                struct rte_flow_item_eth *eth_spec,
7033                struct rte_flow_item_eth *eth_mask)
7034 {
7035         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7036 }
7037
7038 /**
7039  * Create default miss flow rule matching lacp traffic
7040  *
7041  * @param dev
7042  *   Pointer to Ethernet device.
7043  * @param eth_spec
7044  *   An Ethernet flow spec to apply.
7045  *
7046  * @return
7047  *   0 on success, a negative errno value otherwise and rte_errno is set.
7048  */
7049 int
7050 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7051 {
7052         /*
7053          * The LACP matching is done by only using ether type since using
7054          * a multicast dst mac causes kernel to give low priority to this flow.
7055          */
7056         static const struct rte_flow_item_eth lacp_spec = {
7057                 .type = RTE_BE16(0x8809),
7058         };
7059         static const struct rte_flow_item_eth lacp_mask = {
7060                 .type = 0xffff,
7061         };
7062         const struct rte_flow_attr attr = {
7063                 .ingress = 1,
7064         };
7065         struct rte_flow_item items[] = {
7066                 {
7067                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7068                         .spec = &lacp_spec,
7069                         .mask = &lacp_mask,
7070                 },
7071                 {
7072                         .type = RTE_FLOW_ITEM_TYPE_END,
7073                 },
7074         };
7075         struct rte_flow_action actions[] = {
7076                 {
7077                         .type = (enum rte_flow_action_type)
7078                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7079                 },
7080                 {
7081                         .type = RTE_FLOW_ACTION_TYPE_END,
7082                 },
7083         };
7084         struct rte_flow_error error;
7085         uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7086                                         &attr, items, actions,
7087                                         false, &error);
7088
7089         if (!flow_idx)
7090                 return -rte_errno;
7091         return 0;
7092 }
7093
7094 /**
7095  * Destroy a flow.
7096  *
7097  * @see rte_flow_destroy()
7098  * @see rte_flow_ops
7099  */
7100 int
7101 mlx5_flow_destroy(struct rte_eth_dev *dev,
7102                   struct rte_flow *flow,
7103                   struct rte_flow_error *error __rte_unused)
7104 {
7105         flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7106                                 (uintptr_t)(void *)flow);
7107         return 0;
7108 }
7109
7110 /**
7111  * Destroy all flows.
7112  *
7113  * @see rte_flow_flush()
7114  * @see rte_flow_ops
7115  */
7116 int
7117 mlx5_flow_flush(struct rte_eth_dev *dev,
7118                 struct rte_flow_error *error __rte_unused)
7119 {
7120         mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7121         return 0;
7122 }
7123
7124 /**
7125  * Isolated mode.
7126  *
7127  * @see rte_flow_isolate()
7128  * @see rte_flow_ops
7129  */
7130 int
7131 mlx5_flow_isolate(struct rte_eth_dev *dev,
7132                   int enable,
7133                   struct rte_flow_error *error)
7134 {
7135         struct mlx5_priv *priv = dev->data->dev_private;
7136
7137         if (dev->data->dev_started) {
7138                 rte_flow_error_set(error, EBUSY,
7139                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7140                                    NULL,
7141                                    "port must be stopped first");
7142                 return -rte_errno;
7143         }
7144         priv->isolated = !!enable;
7145         if (enable)
7146                 dev->dev_ops = &mlx5_dev_ops_isolate;
7147         else
7148                 dev->dev_ops = &mlx5_dev_ops;
7149
7150         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7151         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7152
7153         return 0;
7154 }
7155
7156 /**
7157  * Query a flow.
7158  *
7159  * @see rte_flow_query()
7160  * @see rte_flow_ops
7161  */
7162 static int
7163 flow_drv_query(struct rte_eth_dev *dev,
7164                uint32_t flow_idx,
7165                const struct rte_flow_action *actions,
7166                void *data,
7167                struct rte_flow_error *error)
7168 {
7169         struct mlx5_priv *priv = dev->data->dev_private;
7170         const struct mlx5_flow_driver_ops *fops;
7171         struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7172                                                flow_idx);
7173         enum mlx5_flow_drv_type ftype;
7174
7175         if (!flow) {
7176                 return rte_flow_error_set(error, ENOENT,
7177                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7178                           NULL,
7179                           "invalid flow handle");
7180         }
7181         ftype = flow->drv_type;
7182         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7183         fops = flow_get_drv_ops(ftype);
7184
7185         return fops->query(dev, flow, actions, data, error);
7186 }
7187
7188 /**
7189  * Query a flow.
7190  *
7191  * @see rte_flow_query()
7192  * @see rte_flow_ops
7193  */
7194 int
7195 mlx5_flow_query(struct rte_eth_dev *dev,
7196                 struct rte_flow *flow,
7197                 const struct rte_flow_action *actions,
7198                 void *data,
7199                 struct rte_flow_error *error)
7200 {
7201         int ret;
7202
7203         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7204                              error);
7205         if (ret < 0)
7206                 return ret;
7207         return 0;
7208 }
7209
7210 /**
7211  * Get rte_flow callbacks.
7212  *
7213  * @param dev
7214  *   Pointer to Ethernet device structure.
7215  * @param ops
7216  *   Pointer to operation-specific structure.
7217  *
7218  * @return 0
7219  */
7220 int
7221 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7222                   const struct rte_flow_ops **ops)
7223 {
7224         *ops = &mlx5_flow_ops;
7225         return 0;
7226 }
7227
7228 /**
7229  * Validate meter policy actions.
7230  * Dispatcher for action type specific validation.
7231  *
7232  * @param[in] dev
7233  *   Pointer to the Ethernet device structure.
7234  * @param[in] action
7235  *   The meter policy action object to validate.
7236  * @param[in] attr
7237  *   Attributes of flow to determine steering domain.
7238  * @param[out] is_rss
7239  *   Is RSS or not.
7240  * @param[out] domain_bitmap
7241  *   Domain bitmap.
7242  * @param[out] is_def_policy
7243  *   Is default policy or not.
7244  * @param[out] error
7245  *   Perform verbose error reporting if not NULL. Initialized in case of
7246  *   error only.
7247  *
7248  * @return
7249  *   0 on success, otherwise negative errno value.
7250  */
7251 int
7252 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7253                         const struct rte_flow_action *actions[RTE_COLORS],
7254                         struct rte_flow_attr *attr,
7255                         bool *is_rss,
7256                         uint8_t *domain_bitmap,
7257                         uint8_t *policy_mode,
7258                         struct rte_mtr_error *error)
7259 {
7260         const struct mlx5_flow_driver_ops *fops;
7261
7262         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7263         return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7264                                        domain_bitmap, policy_mode, error);
7265 }
7266
7267 /**
7268  * Destroy the meter table set.
7269  *
7270  * @param[in] dev
7271  *   Pointer to Ethernet device.
7272  * @param[in] mtr_policy
7273  *   Meter policy struct.
7274  */
7275 void
7276 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7277                       struct mlx5_flow_meter_policy *mtr_policy)
7278 {
7279         const struct mlx5_flow_driver_ops *fops;
7280
7281         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7282         fops->destroy_mtr_acts(dev, mtr_policy);
7283 }
7284
7285 /**
7286  * Create policy action, lock free,
7287  * (mutex should be acquired by caller).
7288  * Dispatcher for action type specific call.
7289  *
7290  * @param[in] dev
7291  *   Pointer to the Ethernet device structure.
7292  * @param[in] mtr_policy
7293  *   Meter policy struct.
7294  * @param[in] action
7295  *   Action specification used to create meter actions.
7296  * @param[out] error
7297  *   Perform verbose error reporting if not NULL. Initialized in case of
7298  *   error only.
7299  *
7300  * @return
7301  *   0 on success, otherwise negative errno value.
7302  */
7303 int
7304 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7305                       struct mlx5_flow_meter_policy *mtr_policy,
7306                       const struct rte_flow_action *actions[RTE_COLORS],
7307                       struct rte_mtr_error *error)
7308 {
7309         const struct mlx5_flow_driver_ops *fops;
7310
7311         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7312         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7313 }
7314
7315 /**
7316  * Create policy rules, lock free,
7317  * (mutex should be acquired by caller).
7318  * Dispatcher for action type specific call.
7319  *
7320  * @param[in] dev
7321  *   Pointer to the Ethernet device structure.
7322  * @param[in] mtr_policy
7323  *   Meter policy struct.
7324  *
7325  * @return
7326  *   0 on success, -1 otherwise.
7327  */
7328 int
7329 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7330                              struct mlx5_flow_meter_policy *mtr_policy)
7331 {
7332         const struct mlx5_flow_driver_ops *fops;
7333
7334         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7335         return fops->create_policy_rules(dev, mtr_policy);
7336 }
7337
7338 /**
7339  * Destroy policy rules, lock free,
7340  * (mutex should be acquired by caller).
7341  * Dispatcher for action type specific call.
7342  *
7343  * @param[in] dev
7344  *   Pointer to the Ethernet device structure.
7345  * @param[in] mtr_policy
7346  *   Meter policy struct.
7347  */
7348 void
7349 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7350                              struct mlx5_flow_meter_policy *mtr_policy)
7351 {
7352         const struct mlx5_flow_driver_ops *fops;
7353
7354         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7355         fops->destroy_policy_rules(dev, mtr_policy);
7356 }
7357
7358 /**
7359  * Destroy the default policy table set.
7360  *
7361  * @param[in] dev
7362  *   Pointer to Ethernet device.
7363  */
7364 void
7365 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7366 {
7367         const struct mlx5_flow_driver_ops *fops;
7368
7369         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7370         fops->destroy_def_policy(dev);
7371 }
7372
7373 /**
7374  * Destroy the default policy table set.
7375  *
7376  * @param[in] dev
7377  *   Pointer to Ethernet device.
7378  *
7379  * @return
7380  *   0 on success, -1 otherwise.
7381  */
7382 int
7383 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7384 {
7385         const struct mlx5_flow_driver_ops *fops;
7386
7387         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7388         return fops->create_def_policy(dev);
7389 }
7390
7391 /**
7392  * Create the needed meter and suffix tables.
7393  *
7394  * @param[in] dev
7395  *   Pointer to Ethernet device.
7396  *
7397  * @return
7398  *   0 on success, -1 otherwise.
7399  */
7400 int
7401 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7402                         struct mlx5_flow_meter_info *fm,
7403                         uint32_t mtr_idx,
7404                         uint8_t domain_bitmap)
7405 {
7406         const struct mlx5_flow_driver_ops *fops;
7407
7408         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7409         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7410 }
7411
7412 /**
7413  * Destroy the meter table set.
7414  *
7415  * @param[in] dev
7416  *   Pointer to Ethernet device.
7417  * @param[in] tbl
7418  *   Pointer to the meter table set.
7419  */
7420 void
7421 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7422                            struct mlx5_flow_meter_info *fm)
7423 {
7424         const struct mlx5_flow_driver_ops *fops;
7425
7426         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7427         fops->destroy_mtr_tbls(dev, fm);
7428 }
7429
7430 /**
7431  * Destroy the global meter drop table.
7432  *
7433  * @param[in] dev
7434  *   Pointer to Ethernet device.
7435  */
7436 void
7437 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7438 {
7439         const struct mlx5_flow_driver_ops *fops;
7440
7441         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7442         fops->destroy_mtr_drop_tbls(dev);
7443 }
7444
7445 /**
7446  * Destroy the sub policy table with RX queue.
7447  *
7448  * @param[in] dev
7449  *   Pointer to Ethernet device.
7450  * @param[in] mtr_policy
7451  *   Pointer to meter policy table.
7452  */
7453 void
7454 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
7455                 struct mlx5_flow_meter_policy *mtr_policy)
7456 {
7457         const struct mlx5_flow_driver_ops *fops;
7458
7459         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7460         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
7461 }
7462
7463 /**
7464  * Allocate the needed aso flow meter id.
7465  *
7466  * @param[in] dev
7467  *   Pointer to Ethernet device.
7468  *
7469  * @return
7470  *   Index to aso flow meter on success, NULL otherwise.
7471  */
7472 uint32_t
7473 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7474 {
7475         const struct mlx5_flow_driver_ops *fops;
7476
7477         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7478         return fops->create_meter(dev);
7479 }
7480
7481 /**
7482  * Free the aso flow meter id.
7483  *
7484  * @param[in] dev
7485  *   Pointer to Ethernet device.
7486  * @param[in] mtr_idx
7487  *  Index to aso flow meter to be free.
7488  *
7489  * @return
7490  *   0 on success.
7491  */
7492 void
7493 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7494 {
7495         const struct mlx5_flow_driver_ops *fops;
7496
7497         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7498         fops->free_meter(dev, mtr_idx);
7499 }
7500
7501 /**
7502  * Allocate a counter.
7503  *
7504  * @param[in] dev
7505  *   Pointer to Ethernet device structure.
7506  *
7507  * @return
7508  *   Index to allocated counter  on success, 0 otherwise.
7509  */
7510 uint32_t
7511 mlx5_counter_alloc(struct rte_eth_dev *dev)
7512 {
7513         const struct mlx5_flow_driver_ops *fops;
7514         struct rte_flow_attr attr = { .transfer = 0 };
7515
7516         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7517                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7518                 return fops->counter_alloc(dev);
7519         }
7520         DRV_LOG(ERR,
7521                 "port %u counter allocate is not supported.",
7522                  dev->data->port_id);
7523         return 0;
7524 }
7525
7526 /**
7527  * Free a counter.
7528  *
7529  * @param[in] dev
7530  *   Pointer to Ethernet device structure.
7531  * @param[in] cnt
7532  *   Index to counter to be free.
7533  */
7534 void
7535 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7536 {
7537         const struct mlx5_flow_driver_ops *fops;
7538         struct rte_flow_attr attr = { .transfer = 0 };
7539
7540         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7541                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7542                 fops->counter_free(dev, cnt);
7543                 return;
7544         }
7545         DRV_LOG(ERR,
7546                 "port %u counter free is not supported.",
7547                  dev->data->port_id);
7548 }
7549
7550 /**
7551  * Query counter statistics.
7552  *
7553  * @param[in] dev
7554  *   Pointer to Ethernet device structure.
7555  * @param[in] cnt
7556  *   Index to counter to query.
7557  * @param[in] clear
7558  *   Set to clear counter statistics.
7559  * @param[out] pkts
7560  *   The counter hits packets number to save.
7561  * @param[out] bytes
7562  *   The counter hits bytes number to save.
7563  *
7564  * @return
7565  *   0 on success, a negative errno value otherwise.
7566  */
7567 int
7568 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7569                    bool clear, uint64_t *pkts, uint64_t *bytes)
7570 {
7571         const struct mlx5_flow_driver_ops *fops;
7572         struct rte_flow_attr attr = { .transfer = 0 };
7573
7574         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7575                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7576                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7577         }
7578         DRV_LOG(ERR,
7579                 "port %u counter query is not supported.",
7580                  dev->data->port_id);
7581         return -ENOTSUP;
7582 }
7583
7584 /**
7585  * Allocate a new memory for the counter values wrapped by all the needed
7586  * management.
7587  *
7588  * @param[in] sh
7589  *   Pointer to mlx5_dev_ctx_shared object.
7590  *
7591  * @return
7592  *   0 on success, a negative errno value otherwise.
7593  */
7594 static int
7595 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7596 {
7597         struct mlx5_devx_mkey_attr mkey_attr;
7598         struct mlx5_counter_stats_mem_mng *mem_mng;
7599         volatile struct flow_counter_stats *raw_data;
7600         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7601         int size = (sizeof(struct flow_counter_stats) *
7602                         MLX5_COUNTERS_PER_POOL +
7603                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7604                         sizeof(struct mlx5_counter_stats_mem_mng);
7605         size_t pgsize = rte_mem_page_size();
7606         uint8_t *mem;
7607         int i;
7608
7609         if (pgsize == (size_t)-1) {
7610                 DRV_LOG(ERR, "Failed to get mem page size");
7611                 rte_errno = ENOMEM;
7612                 return -ENOMEM;
7613         }
7614         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7615         if (!mem) {
7616                 rte_errno = ENOMEM;
7617                 return -ENOMEM;
7618         }
7619         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7620         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7621         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7622                                                  IBV_ACCESS_LOCAL_WRITE);
7623         if (!mem_mng->umem) {
7624                 rte_errno = errno;
7625                 mlx5_free(mem);
7626                 return -rte_errno;
7627         }
7628         memset(&mkey_attr, 0, sizeof(mkey_attr));
7629         mkey_attr.addr = (uintptr_t)mem;
7630         mkey_attr.size = size;
7631         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7632         mkey_attr.pd = sh->pdn;
7633         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7634         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7635         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7636         if (!mem_mng->dm) {
7637                 mlx5_os_umem_dereg(mem_mng->umem);
7638                 rte_errno = errno;
7639                 mlx5_free(mem);
7640                 return -rte_errno;
7641         }
7642         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7643         raw_data = (volatile struct flow_counter_stats *)mem;
7644         for (i = 0; i < raws_n; ++i) {
7645                 mem_mng->raws[i].mem_mng = mem_mng;
7646                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7647         }
7648         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7649                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7650                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7651                                  next);
7652         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7653         sh->cmng.mem_mng = mem_mng;
7654         return 0;
7655 }
7656
7657 /**
7658  * Set the statistic memory to the new counter pool.
7659  *
7660  * @param[in] sh
7661  *   Pointer to mlx5_dev_ctx_shared object.
7662  * @param[in] pool
7663  *   Pointer to the pool to set the statistic memory.
7664  *
7665  * @return
7666  *   0 on success, a negative errno value otherwise.
7667  */
7668 static int
7669 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7670                                struct mlx5_flow_counter_pool *pool)
7671 {
7672         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7673         /* Resize statistic memory once used out. */
7674         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7675             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7676                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7677                 return -1;
7678         }
7679         rte_spinlock_lock(&pool->sl);
7680         pool->raw = cmng->mem_mng->raws + pool->index %
7681                     MLX5_CNT_CONTAINER_RESIZE;
7682         rte_spinlock_unlock(&pool->sl);
7683         pool->raw_hw = NULL;
7684         return 0;
7685 }
7686
7687 #define MLX5_POOL_QUERY_FREQ_US 1000000
7688
7689 /**
7690  * Set the periodic procedure for triggering asynchronous batch queries for all
7691  * the counter pools.
7692  *
7693  * @param[in] sh
7694  *   Pointer to mlx5_dev_ctx_shared object.
7695  */
7696 void
7697 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7698 {
7699         uint32_t pools_n, us;
7700
7701         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7702         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7703         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7704         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7705                 sh->cmng.query_thread_on = 0;
7706                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7707         } else {
7708                 sh->cmng.query_thread_on = 1;
7709         }
7710 }
7711
7712 /**
7713  * The periodic procedure for triggering asynchronous batch queries for all the
7714  * counter pools. This function is probably called by the host thread.
7715  *
7716  * @param[in] arg
7717  *   The parameter for the alarm process.
7718  */
7719 void
7720 mlx5_flow_query_alarm(void *arg)
7721 {
7722         struct mlx5_dev_ctx_shared *sh = arg;
7723         int ret;
7724         uint16_t pool_index = sh->cmng.pool_index;
7725         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7726         struct mlx5_flow_counter_pool *pool;
7727         uint16_t n_valid;
7728
7729         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7730                 goto set_alarm;
7731         rte_spinlock_lock(&cmng->pool_update_sl);
7732         pool = cmng->pools[pool_index];
7733         n_valid = cmng->n_valid;
7734         rte_spinlock_unlock(&cmng->pool_update_sl);
7735         /* Set the statistic memory to the new created pool. */
7736         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7737                 goto set_alarm;
7738         if (pool->raw_hw)
7739                 /* There is a pool query in progress. */
7740                 goto set_alarm;
7741         pool->raw_hw =
7742                 LIST_FIRST(&sh->cmng.free_stat_raws);
7743         if (!pool->raw_hw)
7744                 /* No free counter statistics raw memory. */
7745                 goto set_alarm;
7746         /*
7747          * Identify the counters released between query trigger and query
7748          * handle more efficiently. The counter released in this gap period
7749          * should wait for a new round of query as the new arrived packets
7750          * will not be taken into account.
7751          */
7752         pool->query_gen++;
7753         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7754                                                MLX5_COUNTERS_PER_POOL,
7755                                                NULL, NULL,
7756                                                pool->raw_hw->mem_mng->dm->id,
7757                                                (void *)(uintptr_t)
7758                                                pool->raw_hw->data,
7759                                                sh->devx_comp,
7760                                                (uint64_t)(uintptr_t)pool);
7761         if (ret) {
7762                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7763                         " %d", pool->min_dcs->id);
7764                 pool->raw_hw = NULL;
7765                 goto set_alarm;
7766         }
7767         LIST_REMOVE(pool->raw_hw, next);
7768         sh->cmng.pending_queries++;
7769         pool_index++;
7770         if (pool_index >= n_valid)
7771                 pool_index = 0;
7772 set_alarm:
7773         sh->cmng.pool_index = pool_index;
7774         mlx5_set_query_alarm(sh);
7775 }
7776
7777 /**
7778  * Check and callback event for new aged flow in the counter pool
7779  *
7780  * @param[in] sh
7781  *   Pointer to mlx5_dev_ctx_shared object.
7782  * @param[in] pool
7783  *   Pointer to Current counter pool.
7784  */
7785 static void
7786 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7787                    struct mlx5_flow_counter_pool *pool)
7788 {
7789         struct mlx5_priv *priv;
7790         struct mlx5_flow_counter *cnt;
7791         struct mlx5_age_info *age_info;
7792         struct mlx5_age_param *age_param;
7793         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7794         struct mlx5_counter_stats_raw *prev = pool->raw;
7795         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7796         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7797         uint16_t expected = AGE_CANDIDATE;
7798         uint32_t i;
7799
7800         pool->time_of_last_age_check = curr_time;
7801         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7802                 cnt = MLX5_POOL_GET_CNT(pool, i);
7803                 age_param = MLX5_CNT_TO_AGE(cnt);
7804                 if (__atomic_load_n(&age_param->state,
7805                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7806                         continue;
7807                 if (cur->data[i].hits != prev->data[i].hits) {
7808                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7809                                          __ATOMIC_RELAXED);
7810                         continue;
7811                 }
7812                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7813                                        time_delta,
7814                                        __ATOMIC_RELAXED) <= age_param->timeout)
7815                         continue;
7816                 /**
7817                  * Hold the lock first, or if between the
7818                  * state AGE_TMOUT and tailq operation the
7819                  * release happened, the release procedure
7820                  * may delete a non-existent tailq node.
7821                  */
7822                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7823                 age_info = GET_PORT_AGE_INFO(priv);
7824                 rte_spinlock_lock(&age_info->aged_sl);
7825                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7826                                                 AGE_TMOUT, false,
7827                                                 __ATOMIC_RELAXED,
7828                                                 __ATOMIC_RELAXED)) {
7829                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7830                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7831                 }
7832                 rte_spinlock_unlock(&age_info->aged_sl);
7833         }
7834         mlx5_age_event_prepare(sh);
7835 }
7836
7837 /**
7838  * Handler for the HW respond about ready values from an asynchronous batch
7839  * query. This function is probably called by the host thread.
7840  *
7841  * @param[in] sh
7842  *   The pointer to the shared device context.
7843  * @param[in] async_id
7844  *   The Devx async ID.
7845  * @param[in] status
7846  *   The status of the completion.
7847  */
7848 void
7849 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7850                                   uint64_t async_id, int status)
7851 {
7852         struct mlx5_flow_counter_pool *pool =
7853                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7854         struct mlx5_counter_stats_raw *raw_to_free;
7855         uint8_t query_gen = pool->query_gen ^ 1;
7856         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7857         enum mlx5_counter_type cnt_type =
7858                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7859                                 MLX5_COUNTER_TYPE_ORIGIN;
7860
7861         if (unlikely(status)) {
7862                 raw_to_free = pool->raw_hw;
7863         } else {
7864                 raw_to_free = pool->raw;
7865                 if (pool->is_aged)
7866                         mlx5_flow_aging_check(sh, pool);
7867                 rte_spinlock_lock(&pool->sl);
7868                 pool->raw = pool->raw_hw;
7869                 rte_spinlock_unlock(&pool->sl);
7870                 /* Be sure the new raw counters data is updated in memory. */
7871                 rte_io_wmb();
7872                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7873                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7874                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7875                                      &pool->counters[query_gen], next);
7876                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7877                 }
7878         }
7879         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7880         pool->raw_hw = NULL;
7881         sh->cmng.pending_queries--;
7882 }
7883
7884 static int
7885 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7886                     const struct flow_grp_info *grp_info,
7887                     struct rte_flow_error *error)
7888 {
7889         if (grp_info->transfer && grp_info->external &&
7890             grp_info->fdb_def_rule) {
7891                 if (group == UINT32_MAX)
7892                         return rte_flow_error_set
7893                                                 (error, EINVAL,
7894                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7895                                                  NULL,
7896                                                  "group index not supported");
7897                 *table = group + 1;
7898         } else {
7899                 *table = group;
7900         }
7901         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7902         return 0;
7903 }
7904
7905 /**
7906  * Translate the rte_flow group index to HW table value.
7907  *
7908  * If tunnel offload is disabled, all group ids converted to flow table
7909  * id using the standard method.
7910  * If tunnel offload is enabled, group id can be converted using the
7911  * standard or tunnel conversion method. Group conversion method
7912  * selection depends on flags in `grp_info` parameter:
7913  * - Internal (grp_info.external == 0) groups conversion uses the
7914  *   standard method.
7915  * - Group ids in JUMP action converted with the tunnel conversion.
7916  * - Group id in rule attribute conversion depends on a rule type and
7917  *   group id value:
7918  *   ** non zero group attributes converted with the tunnel method
7919  *   ** zero group attribute in non-tunnel rule is converted using the
7920  *      standard method - there's only one root table
7921  *   ** zero group attribute in steer tunnel rule is converted with the
7922  *      standard method - single root table
7923  *   ** zero group attribute in match tunnel rule is a special OvS
7924  *      case: that value is used for portability reasons. That group
7925  *      id is converted with the tunnel conversion method.
7926  *
7927  * @param[in] dev
7928  *   Port device
7929  * @param[in] tunnel
7930  *   PMD tunnel offload object
7931  * @param[in] group
7932  *   rte_flow group index value.
7933  * @param[out] table
7934  *   HW table value.
7935  * @param[in] grp_info
7936  *   flags used for conversion
7937  * @param[out] error
7938  *   Pointer to error structure.
7939  *
7940  * @return
7941  *   0 on success, a negative errno value otherwise and rte_errno is set.
7942  */
7943 int
7944 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7945                          const struct mlx5_flow_tunnel *tunnel,
7946                          uint32_t group, uint32_t *table,
7947                          const struct flow_grp_info *grp_info,
7948                          struct rte_flow_error *error)
7949 {
7950         int ret;
7951         bool standard_translation;
7952
7953         if (!grp_info->skip_scale && grp_info->external &&
7954             group < MLX5_MAX_TABLES_EXTERNAL)
7955                 group *= MLX5_FLOW_TABLE_FACTOR;
7956         if (is_tunnel_offload_active(dev)) {
7957                 standard_translation = !grp_info->external ||
7958                                         grp_info->std_tbl_fix;
7959         } else {
7960                 standard_translation = true;
7961         }
7962         DRV_LOG(DEBUG,
7963                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7964                 dev->data->port_id, group, grp_info->transfer,
7965                 grp_info->external, grp_info->fdb_def_rule,
7966                 standard_translation ? "STANDARD" : "TUNNEL");
7967         if (standard_translation)
7968                 ret = flow_group_to_table(dev->data->port_id, group, table,
7969                                           grp_info, error);
7970         else
7971                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7972                                                       table, error);
7973
7974         return ret;
7975 }
7976
7977 /**
7978  * Discover availability of metadata reg_c's.
7979  *
7980  * Iteratively use test flows to check availability.
7981  *
7982  * @param[in] dev
7983  *   Pointer to the Ethernet device structure.
7984  *
7985  * @return
7986  *   0 on success, a negative errno value otherwise and rte_errno is set.
7987  */
7988 int
7989 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7990 {
7991         struct mlx5_priv *priv = dev->data->dev_private;
7992         struct mlx5_dev_config *config = &priv->config;
7993         enum modify_reg idx;
7994         int n = 0;
7995
7996         /* reg_c[0] and reg_c[1] are reserved. */
7997         config->flow_mreg_c[n++] = REG_C_0;
7998         config->flow_mreg_c[n++] = REG_C_1;
7999         /* Discover availability of other reg_c's. */
8000         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
8001                 struct rte_flow_attr attr = {
8002                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
8003                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
8004                         .ingress = 1,
8005                 };
8006                 struct rte_flow_item items[] = {
8007                         [0] = {
8008                                 .type = RTE_FLOW_ITEM_TYPE_END,
8009                         },
8010                 };
8011                 struct rte_flow_action actions[] = {
8012                         [0] = {
8013                                 .type = (enum rte_flow_action_type)
8014                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
8015                                 .conf = &(struct mlx5_flow_action_copy_mreg){
8016                                         .src = REG_C_1,
8017                                         .dst = idx,
8018                                 },
8019                         },
8020                         [1] = {
8021                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
8022                                 .conf = &(struct rte_flow_action_jump){
8023                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
8024                                 },
8025                         },
8026                         [2] = {
8027                                 .type = RTE_FLOW_ACTION_TYPE_END,
8028                         },
8029                 };
8030                 uint32_t flow_idx;
8031                 struct rte_flow *flow;
8032                 struct rte_flow_error error;
8033
8034                 if (!config->dv_flow_en)
8035                         break;
8036                 /* Create internal flow, validation skips copy action. */
8037                 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
8038                                         items, actions, false, &error);
8039                 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8040                                       flow_idx);
8041                 if (!flow)
8042                         continue;
8043                 config->flow_mreg_c[n++] = idx;
8044                 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
8045         }
8046         for (; n < MLX5_MREG_C_NUM; ++n)
8047                 config->flow_mreg_c[n] = REG_NON;
8048         return 0;
8049 }
8050
8051 int
8052 save_dump_file(const uint8_t *data, uint32_t size,
8053         uint32_t type, uint32_t id, void *arg, FILE *file)
8054 {
8055         char line[BUF_SIZE];
8056         uint32_t out = 0;
8057         uint32_t k;
8058         uint32_t actions_num;
8059         struct rte_flow_query_count *count;
8060
8061         memset(line, 0, BUF_SIZE);
8062         switch (type) {
8063         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
8064                 actions_num = *(uint32_t *)(arg);
8065                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,",
8066                                 type, id, actions_num);
8067                 break;
8068         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
8069                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,",
8070                                 type, id);
8071                 break;
8072         case DR_DUMP_REC_TYPE_PMD_COUNTER:
8073                 count = (struct rte_flow_query_count *)arg;
8074                 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type,
8075                                 id, count->hits, count->bytes);
8076                 return 0;
8077         default:
8078                 return -1;
8079         }
8080
8081         for (k = 0; k < size; k++) {
8082                 /* Make sure we do not overrun the line buffer length. */
8083                 if (out >= BUF_SIZE - 4) {
8084                         line[out] = '\0';
8085                         break;
8086                 }
8087                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
8088                                 (data[k]) & 0xff);
8089         }
8090         fprintf(file, "%s\n", line);
8091         return 0;
8092 }
8093
8094 int
8095 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
8096         struct rte_flow_query_count *count, struct rte_flow_error *error)
8097 {
8098         struct rte_flow_action action[2];
8099         enum mlx5_flow_drv_type ftype;
8100         const struct mlx5_flow_driver_ops *fops;
8101
8102         if (!flow) {
8103                 return rte_flow_error_set(error, ENOENT,
8104                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8105                                 NULL,
8106                                 "invalid flow handle");
8107         }
8108         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
8109         action[1].type = RTE_FLOW_ACTION_TYPE_END;
8110         if (flow->counter) {
8111                 memset(count, 0, sizeof(struct rte_flow_query_count));
8112                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
8113                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
8114                                                 ftype < MLX5_FLOW_TYPE_MAX);
8115                 fops = flow_get_drv_ops(ftype);
8116                 return fops->query(dev, flow, action, count, error);
8117         }
8118         return -1;
8119 }
8120
8121 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8122 /**
8123  * Dump flow ipool data to file
8124  *
8125  * @param[in] dev
8126  *   The pointer to Ethernet device.
8127  * @param[in] file
8128  *   A pointer to a file for output.
8129  * @param[out] error
8130  *   Perform verbose error reporting if not NULL. PMDs initialize this
8131  *   structure in case of error only.
8132  * @return
8133  *   0 on success, a negative value otherwise.
8134  */
8135 int
8136 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
8137         struct rte_flow *flow, FILE *file,
8138         struct rte_flow_error *error)
8139 {
8140         struct mlx5_priv *priv = dev->data->dev_private;
8141         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
8142         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
8143         uint32_t handle_idx;
8144         struct mlx5_flow_handle *dh;
8145         struct rte_flow_query_count count;
8146         uint32_t actions_num;
8147         const uint8_t *data;
8148         size_t size;
8149         uint32_t id;
8150         uint32_t type;
8151
8152         if (!flow) {
8153                 return rte_flow_error_set(error, ENOENT,
8154                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8155                         NULL,
8156                         "invalid flow handle");
8157         }
8158         handle_idx = flow->dev_handles;
8159         while (handle_idx) {
8160                 dh = mlx5_ipool_get(priv->sh->ipool
8161                         [MLX5_IPOOL_MLX5_FLOW], handle_idx);
8162                 if (!dh)
8163                         continue;
8164                 handle_idx = dh->next.next;
8165                 id = (uint32_t)(uintptr_t)dh->drv_flow;
8166
8167                 /* query counter */
8168                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
8169                 if (!mlx5_flow_query_counter(dev, flow, &count, error))
8170                         save_dump_file(NULL, 0, type,
8171                                         id, (void *)&count, file);
8172
8173                 /* Get modify_hdr and encap_decap buf from ipools. */
8174                 encap_decap = NULL;
8175                 modify_hdr = dh->dvh.modify_hdr;
8176
8177                 if (dh->dvh.rix_encap_decap) {
8178                         encap_decap = mlx5_ipool_get(priv->sh->ipool
8179                                                 [MLX5_IPOOL_DECAP_ENCAP],
8180                                                 dh->dvh.rix_encap_decap);
8181                 }
8182                 if (modify_hdr) {
8183                         data = (const uint8_t *)modify_hdr->actions;
8184                         size = (size_t)(modify_hdr->actions_num) * 8;
8185                         actions_num = modify_hdr->actions_num;
8186                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
8187                         save_dump_file(data, size, type, id,
8188                                         (void *)(&actions_num), file);
8189                 }
8190                 if (encap_decap) {
8191                         data = encap_decap->buf;
8192                         size = encap_decap->size;
8193                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
8194                         save_dump_file(data, size, type,
8195                                                 id, NULL, file);
8196                 }
8197         }
8198         return 0;
8199 }
8200 #endif
8201
8202 /**
8203  * Dump flow raw hw data to file
8204  *
8205  * @param[in] dev
8206  *    The pointer to Ethernet device.
8207  * @param[in] file
8208  *   A pointer to a file for output.
8209  * @param[out] error
8210  *   Perform verbose error reporting if not NULL. PMDs initialize this
8211  *   structure in case of error only.
8212  * @return
8213  *   0 on success, a nagative value otherwise.
8214  */
8215 int
8216 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
8217                    FILE *file,
8218                    struct rte_flow_error *error __rte_unused)
8219 {
8220         struct mlx5_priv *priv = dev->data->dev_private;
8221         struct mlx5_dev_ctx_shared *sh = priv->sh;
8222         uint32_t handle_idx;
8223         int ret;
8224         struct mlx5_flow_handle *dh;
8225         struct rte_flow *flow;
8226 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8227         uint32_t idx;
8228 #endif
8229
8230         if (!priv->config.dv_flow_en) {
8231                 if (fputs("device dv flow disabled\n", file) <= 0)
8232                         return -errno;
8233                 return -ENOTSUP;
8234         }
8235
8236         /* dump all */
8237         if (!flow_idx) {
8238 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8239                 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow)
8240                         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8241 #endif
8242                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
8243                                         sh->rx_domain,
8244                                         sh->tx_domain, file);
8245         }
8246         /* dump one */
8247         flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8248                         (uintptr_t)(void *)flow_idx);
8249         if (!flow)
8250                 return -ENOENT;
8251
8252 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8253         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8254 #endif
8255         handle_idx = flow->dev_handles;
8256         while (handle_idx) {
8257                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8258                                 handle_idx);
8259                 if (!dh)
8260                         return -ENOENT;
8261                 if (dh->drv_flow) {
8262                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
8263                                         file);
8264                         if (ret)
8265                                 return -ENOENT;
8266                 }
8267                 handle_idx = dh->next.next;
8268         }
8269         return 0;
8270 }
8271
8272 /**
8273  * Get aged-out flows.
8274  *
8275  * @param[in] dev
8276  *   Pointer to the Ethernet device structure.
8277  * @param[in] context
8278  *   The address of an array of pointers to the aged-out flows contexts.
8279  * @param[in] nb_countexts
8280  *   The length of context array pointers.
8281  * @param[out] error
8282  *   Perform verbose error reporting if not NULL. Initialized in case of
8283  *   error only.
8284  *
8285  * @return
8286  *   how many contexts get in success, otherwise negative errno value.
8287  *   if nb_contexts is 0, return the amount of all aged contexts.
8288  *   if nb_contexts is not 0 , return the amount of aged flows reported
8289  *   in the context array.
8290  */
8291 int
8292 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
8293                         uint32_t nb_contexts, struct rte_flow_error *error)
8294 {
8295         const struct mlx5_flow_driver_ops *fops;
8296         struct rte_flow_attr attr = { .transfer = 0 };
8297
8298         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8299                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8300                 return fops->get_aged_flows(dev, contexts, nb_contexts,
8301                                                     error);
8302         }
8303         DRV_LOG(ERR,
8304                 "port %u get aged flows is not supported.",
8305                  dev->data->port_id);
8306         return -ENOTSUP;
8307 }
8308
8309 /* Wrapper for driver action_validate op callback */
8310 static int
8311 flow_drv_action_validate(struct rte_eth_dev *dev,
8312                          const struct rte_flow_indir_action_conf *conf,
8313                          const struct rte_flow_action *action,
8314                          const struct mlx5_flow_driver_ops *fops,
8315                          struct rte_flow_error *error)
8316 {
8317         static const char err_msg[] = "indirect action validation unsupported";
8318
8319         if (!fops->action_validate) {
8320                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8321                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8322                                    NULL, err_msg);
8323                 return -rte_errno;
8324         }
8325         return fops->action_validate(dev, conf, action, error);
8326 }
8327
8328 /**
8329  * Destroys the shared action by handle.
8330  *
8331  * @param dev
8332  *   Pointer to Ethernet device structure.
8333  * @param[in] handle
8334  *   Handle for the indirect action object to be destroyed.
8335  * @param[out] error
8336  *   Perform verbose error reporting if not NULL. PMDs initialize this
8337  *   structure in case of error only.
8338  *
8339  * @return
8340  *   0 on success, a negative errno value otherwise and rte_errno is set.
8341  *
8342  * @note: wrapper for driver action_create op callback.
8343  */
8344 static int
8345 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
8346                            struct rte_flow_action_handle *handle,
8347                            struct rte_flow_error *error)
8348 {
8349         static const char err_msg[] = "indirect action destruction unsupported";
8350         struct rte_flow_attr attr = { .transfer = 0 };
8351         const struct mlx5_flow_driver_ops *fops =
8352                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8353
8354         if (!fops->action_destroy) {
8355                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8356                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8357                                    NULL, err_msg);
8358                 return -rte_errno;
8359         }
8360         return fops->action_destroy(dev, handle, error);
8361 }
8362
8363 /* Wrapper for driver action_destroy op callback */
8364 static int
8365 flow_drv_action_update(struct rte_eth_dev *dev,
8366                        struct rte_flow_action_handle *handle,
8367                        const void *update,
8368                        const struct mlx5_flow_driver_ops *fops,
8369                        struct rte_flow_error *error)
8370 {
8371         static const char err_msg[] = "indirect action update unsupported";
8372
8373         if (!fops->action_update) {
8374                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8375                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8376                                    NULL, err_msg);
8377                 return -rte_errno;
8378         }
8379         return fops->action_update(dev, handle, update, error);
8380 }
8381
8382 /* Wrapper for driver action_destroy op callback */
8383 static int
8384 flow_drv_action_query(struct rte_eth_dev *dev,
8385                       const struct rte_flow_action_handle *handle,
8386                       void *data,
8387                       const struct mlx5_flow_driver_ops *fops,
8388                       struct rte_flow_error *error)
8389 {
8390         static const char err_msg[] = "indirect action query unsupported";
8391
8392         if (!fops->action_query) {
8393                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8394                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8395                                    NULL, err_msg);
8396                 return -rte_errno;
8397         }
8398         return fops->action_query(dev, handle, data, error);
8399 }
8400
8401 /**
8402  * Create indirect action for reuse in multiple flow rules.
8403  *
8404  * @param dev
8405  *   Pointer to Ethernet device structure.
8406  * @param conf
8407  *   Pointer to indirect action object configuration.
8408  * @param[in] action
8409  *   Action configuration for indirect action object creation.
8410  * @param[out] error
8411  *   Perform verbose error reporting if not NULL. PMDs initialize this
8412  *   structure in case of error only.
8413  * @return
8414  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8415  */
8416 static struct rte_flow_action_handle *
8417 mlx5_action_handle_create(struct rte_eth_dev *dev,
8418                           const struct rte_flow_indir_action_conf *conf,
8419                           const struct rte_flow_action *action,
8420                           struct rte_flow_error *error)
8421 {
8422         static const char err_msg[] = "indirect action creation unsupported";
8423         struct rte_flow_attr attr = { .transfer = 0 };
8424         const struct mlx5_flow_driver_ops *fops =
8425                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8426
8427         if (flow_drv_action_validate(dev, conf, action, fops, error))
8428                 return NULL;
8429         if (!fops->action_create) {
8430                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8431                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8432                                    NULL, err_msg);
8433                 return NULL;
8434         }
8435         return fops->action_create(dev, conf, action, error);
8436 }
8437
8438 /**
8439  * Updates inplace the indirect action configuration pointed by *handle*
8440  * with the configuration provided as *update* argument.
8441  * The update of the indirect action configuration effects all flow rules
8442  * reusing the action via handle.
8443  *
8444  * @param dev
8445  *   Pointer to Ethernet device structure.
8446  * @param[in] handle
8447  *   Handle for the indirect action to be updated.
8448  * @param[in] update
8449  *   Action specification used to modify the action pointed by handle.
8450  *   *update* could be of same type with the action pointed by the *handle*
8451  *   handle argument, or some other structures like a wrapper, depending on
8452  *   the indirect action type.
8453  * @param[out] error
8454  *   Perform verbose error reporting if not NULL. PMDs initialize this
8455  *   structure in case of error only.
8456  *
8457  * @return
8458  *   0 on success, a negative errno value otherwise and rte_errno is set.
8459  */
8460 static int
8461 mlx5_action_handle_update(struct rte_eth_dev *dev,
8462                 struct rte_flow_action_handle *handle,
8463                 const void *update,
8464                 struct rte_flow_error *error)
8465 {
8466         struct rte_flow_attr attr = { .transfer = 0 };
8467         const struct mlx5_flow_driver_ops *fops =
8468                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8469         int ret;
8470
8471         ret = flow_drv_action_validate(dev, NULL,
8472                         (const struct rte_flow_action *)update, fops, error);
8473         if (ret)
8474                 return ret;
8475         return flow_drv_action_update(dev, handle, update, fops,
8476                                       error);
8477 }
8478
8479 /**
8480  * Query the indirect action by handle.
8481  *
8482  * This function allows retrieving action-specific data such as counters.
8483  * Data is gathered by special action which may be present/referenced in
8484  * more than one flow rule definition.
8485  *
8486  * see @RTE_FLOW_ACTION_TYPE_COUNT
8487  *
8488  * @param dev
8489  *   Pointer to Ethernet device structure.
8490  * @param[in] handle
8491  *   Handle for the indirect action to query.
8492  * @param[in, out] data
8493  *   Pointer to storage for the associated query data type.
8494  * @param[out] error
8495  *   Perform verbose error reporting if not NULL. PMDs initialize this
8496  *   structure in case of error only.
8497  *
8498  * @return
8499  *   0 on success, a negative errno value otherwise and rte_errno is set.
8500  */
8501 static int
8502 mlx5_action_handle_query(struct rte_eth_dev *dev,
8503                          const struct rte_flow_action_handle *handle,
8504                          void *data,
8505                          struct rte_flow_error *error)
8506 {
8507         struct rte_flow_attr attr = { .transfer = 0 };
8508         const struct mlx5_flow_driver_ops *fops =
8509                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8510
8511         return flow_drv_action_query(dev, handle, data, fops, error);
8512 }
8513
8514 /**
8515  * Destroy all indirect actions (shared RSS).
8516  *
8517  * @param dev
8518  *   Pointer to Ethernet device.
8519  *
8520  * @return
8521  *   0 on success, a negative errno value otherwise and rte_errno is set.
8522  */
8523 int
8524 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8525 {
8526         struct rte_flow_error error;
8527         struct mlx5_priv *priv = dev->data->dev_private;
8528         struct mlx5_shared_action_rss *shared_rss;
8529         int ret = 0;
8530         uint32_t idx;
8531
8532         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8533                       priv->rss_shared_actions, idx, shared_rss, next) {
8534                 ret |= mlx5_action_handle_destroy(dev,
8535                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8536         }
8537         return ret;
8538 }
8539
8540 #ifndef HAVE_MLX5DV_DR
8541 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8542 #else
8543 #define MLX5_DOMAIN_SYNC_FLOW \
8544         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8545 #endif
8546
8547 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8548 {
8549         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8550         const struct mlx5_flow_driver_ops *fops;
8551         int ret;
8552         struct rte_flow_attr attr = { .transfer = 0 };
8553
8554         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8555         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8556         if (ret > 0)
8557                 ret = -ret;
8558         return ret;
8559 }
8560
8561 const struct mlx5_flow_tunnel *
8562 mlx5_get_tof(const struct rte_flow_item *item,
8563              const struct rte_flow_action *action,
8564              enum mlx5_tof_rule_type *rule_type)
8565 {
8566         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8567                 if (item->type == (typeof(item->type))
8568                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8569                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8570                         return flow_items_to_tunnel(item);
8571                 }
8572         }
8573         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8574                 if (action->type == (typeof(action->type))
8575                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8576                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8577                         return flow_actions_to_tunnel(action);
8578                 }
8579         }
8580         return NULL;
8581 }
8582
8583 /**
8584  * tunnel offload functionalilty is defined for DV environment only
8585  */
8586 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8587 __extension__
8588 union tunnel_offload_mark {
8589         uint32_t val;
8590         struct {
8591                 uint32_t app_reserve:8;
8592                 uint32_t table_id:15;
8593                 uint32_t transfer:1;
8594                 uint32_t _unused_:8;
8595         };
8596 };
8597
8598 static bool
8599 mlx5_access_tunnel_offload_db
8600         (struct rte_eth_dev *dev,
8601          bool (*match)(struct rte_eth_dev *,
8602                        struct mlx5_flow_tunnel *, const void *),
8603          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8604          void (*miss)(struct rte_eth_dev *, void *),
8605          void *ctx, bool lock_op);
8606
8607 static int
8608 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8609                              struct rte_flow *flow,
8610                              const struct rte_flow_attr *attr,
8611                              const struct rte_flow_action *app_actions,
8612                              uint32_t flow_idx,
8613                              const struct mlx5_flow_tunnel *tunnel,
8614                              struct tunnel_default_miss_ctx *ctx,
8615                              struct rte_flow_error *error)
8616 {
8617         struct mlx5_priv *priv = dev->data->dev_private;
8618         struct mlx5_flow *dev_flow;
8619         struct rte_flow_attr miss_attr = *attr;
8620         const struct rte_flow_item miss_items[2] = {
8621                 {
8622                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8623                         .spec = NULL,
8624                         .last = NULL,
8625                         .mask = NULL
8626                 },
8627                 {
8628                         .type = RTE_FLOW_ITEM_TYPE_END,
8629                         .spec = NULL,
8630                         .last = NULL,
8631                         .mask = NULL
8632                 }
8633         };
8634         union tunnel_offload_mark mark_id;
8635         struct rte_flow_action_mark miss_mark;
8636         struct rte_flow_action miss_actions[3] = {
8637                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8638                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8639         };
8640         const struct rte_flow_action_jump *jump_data;
8641         uint32_t i, flow_table = 0; /* prevent compilation warning */
8642         struct flow_grp_info grp_info = {
8643                 .external = 1,
8644                 .transfer = attr->transfer,
8645                 .fdb_def_rule = !!priv->fdb_def_rule,
8646                 .std_tbl_fix = 0,
8647         };
8648         int ret;
8649
8650         if (!attr->transfer) {
8651                 uint32_t q_size;
8652
8653                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8654                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8655                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8656                                          0, SOCKET_ID_ANY);
8657                 if (!ctx->queue)
8658                         return rte_flow_error_set
8659                                 (error, ENOMEM,
8660                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8661                                 NULL, "invalid default miss RSS");
8662                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8663                 ctx->action_rss.level = 0,
8664                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8665                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8666                 ctx->action_rss.queue_num = priv->reta_idx_n,
8667                 ctx->action_rss.key = priv->rss_conf.rss_key,
8668                 ctx->action_rss.queue = ctx->queue;
8669                 if (!priv->reta_idx_n || !priv->rxqs_n)
8670                         return rte_flow_error_set
8671                                 (error, EINVAL,
8672                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8673                                 NULL, "invalid port configuration");
8674                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8675                         ctx->action_rss.types = 0;
8676                 for (i = 0; i != priv->reta_idx_n; ++i)
8677                         ctx->queue[i] = (*priv->reta_idx)[i];
8678         } else {
8679                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8680                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8681         }
8682         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8683         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8684         jump_data = app_actions->conf;
8685         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8686         miss_attr.group = jump_data->group;
8687         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8688                                        &flow_table, &grp_info, error);
8689         if (ret)
8690                 return rte_flow_error_set(error, EINVAL,
8691                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8692                                           NULL, "invalid tunnel id");
8693         mark_id.app_reserve = 0;
8694         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8695         mark_id.transfer = !!attr->transfer;
8696         mark_id._unused_ = 0;
8697         miss_mark.id = mark_id.val;
8698         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8699                                     miss_items, miss_actions, flow_idx, error);
8700         if (!dev_flow)
8701                 return -rte_errno;
8702         dev_flow->flow = flow;
8703         dev_flow->external = true;
8704         dev_flow->tunnel = tunnel;
8705         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8706         /* Subflow object was created, we must include one in the list. */
8707         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8708                       dev_flow->handle, next);
8709         DRV_LOG(DEBUG,
8710                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8711                 dev->data->port_id, tunnel->app_tunnel.type,
8712                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8713         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8714                                   miss_actions, error);
8715         if (!ret)
8716                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8717                                                   error);
8718
8719         return ret;
8720 }
8721
8722 static const struct mlx5_flow_tbl_data_entry  *
8723 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8724 {
8725         struct mlx5_priv *priv = dev->data->dev_private;
8726         struct mlx5_dev_ctx_shared *sh = priv->sh;
8727         struct mlx5_list_entry *he;
8728         union tunnel_offload_mark mbits = { .val = mark };
8729         union mlx5_flow_tbl_key table_key = {
8730                 {
8731                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8732                         .id = 0,
8733                         .reserved = 0,
8734                         .dummy = 0,
8735                         .is_fdb = !!mbits.transfer,
8736                         .is_egress = 0,
8737                 }
8738         };
8739         struct mlx5_flow_cb_ctx ctx = {
8740                 .data = &table_key.v64,
8741         };
8742
8743         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
8744         return he ?
8745                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8746 }
8747
8748 static void
8749 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
8750                                    struct mlx5_list_entry *entry)
8751 {
8752         struct mlx5_dev_ctx_shared *sh = tool_ctx;
8753         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8754
8755         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8756                         tunnel_flow_tbl_to_id(tte->flow_table));
8757         mlx5_free(tte);
8758 }
8759
8760 static int
8761 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
8762                                   struct mlx5_list_entry *entry, void *cb_ctx)
8763 {
8764         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
8765         union tunnel_tbl_key tbl = {
8766                 .val = *(uint64_t *)(ctx->data),
8767         };
8768         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8769
8770         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8771 }
8772
8773 static struct mlx5_list_entry *
8774 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
8775 {
8776         struct mlx5_dev_ctx_shared *sh = tool_ctx;
8777         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
8778         struct tunnel_tbl_entry *tte;
8779         union tunnel_tbl_key tbl = {
8780                 .val = *(uint64_t *)(ctx->data),
8781         };
8782
8783         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8784                           sizeof(*tte), 0,
8785                           SOCKET_ID_ANY);
8786         if (!tte)
8787                 goto err;
8788         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8789                           &tte->flow_table);
8790         if (tte->flow_table >= MLX5_MAX_TABLES) {
8791                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8792                         tte->flow_table);
8793                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8794                                 tte->flow_table);
8795                 goto err;
8796         } else if (!tte->flow_table) {
8797                 goto err;
8798         }
8799         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8800         tte->tunnel_id = tbl.tunnel_id;
8801         tte->group = tbl.group;
8802         return &tte->hash;
8803 err:
8804         if (tte)
8805                 mlx5_free(tte);
8806         return NULL;
8807 }
8808
8809 static struct mlx5_list_entry *
8810 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
8811                                   struct mlx5_list_entry *oentry,
8812                                   void *cb_ctx __rte_unused)
8813 {
8814         struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
8815                                                    0, SOCKET_ID_ANY);
8816
8817         if (!tte)
8818                 return NULL;
8819         memcpy(tte, oentry, sizeof(*tte));
8820         return &tte->hash;
8821 }
8822
8823 static void
8824 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
8825                                        struct mlx5_list_entry *entry)
8826 {
8827         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8828
8829         mlx5_free(tte);
8830 }
8831
8832 static uint32_t
8833 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8834                                 const struct mlx5_flow_tunnel *tunnel,
8835                                 uint32_t group, uint32_t *table,
8836                                 struct rte_flow_error *error)
8837 {
8838         struct mlx5_list_entry *he;
8839         struct tunnel_tbl_entry *tte;
8840         union tunnel_tbl_key key = {
8841                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8842                 .group = group
8843         };
8844         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8845         struct mlx5_hlist *group_hash;
8846         struct mlx5_flow_cb_ctx ctx = {
8847                 .data = &key.val,
8848         };
8849
8850         group_hash = tunnel ? tunnel->groups : thub->groups;
8851         he = mlx5_hlist_register(group_hash, key.val, &ctx);
8852         if (!he)
8853                 return rte_flow_error_set(error, EINVAL,
8854                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8855                                           NULL,
8856                                           "tunnel group index not supported");
8857         tte = container_of(he, typeof(*tte), hash);
8858         *table = tte->flow_table;
8859         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8860                 dev->data->port_id, key.tunnel_id, group, *table);
8861         return 0;
8862 }
8863
8864 static void
8865 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8866                       struct mlx5_flow_tunnel *tunnel)
8867 {
8868         struct mlx5_priv *priv = dev->data->dev_private;
8869         struct mlx5_indexed_pool *ipool;
8870
8871         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8872                 dev->data->port_id, tunnel->tunnel_id);
8873         LIST_REMOVE(tunnel, chain);
8874         mlx5_hlist_destroy(tunnel->groups);
8875         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8876         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8877 }
8878
8879 static bool
8880 mlx5_access_tunnel_offload_db
8881         (struct rte_eth_dev *dev,
8882          bool (*match)(struct rte_eth_dev *,
8883                        struct mlx5_flow_tunnel *, const void *),
8884          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8885          void (*miss)(struct rte_eth_dev *, void *),
8886          void *ctx, bool lock_op)
8887 {
8888         bool verdict = false;
8889         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8890         struct mlx5_flow_tunnel *tunnel;
8891
8892         rte_spinlock_lock(&thub->sl);
8893         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8894                 verdict = match(dev, tunnel, (const void *)ctx);
8895                 if (verdict)
8896                         break;
8897         }
8898         if (!lock_op)
8899                 rte_spinlock_unlock(&thub->sl);
8900         if (verdict && hit)
8901                 hit(dev, tunnel, ctx);
8902         if (!verdict && miss)
8903                 miss(dev, ctx);
8904         if (lock_op)
8905                 rte_spinlock_unlock(&thub->sl);
8906
8907         return verdict;
8908 }
8909
8910 struct tunnel_db_find_tunnel_id_ctx {
8911         uint32_t tunnel_id;
8912         struct mlx5_flow_tunnel *tunnel;
8913 };
8914
8915 static bool
8916 find_tunnel_id_match(struct rte_eth_dev *dev,
8917                      struct mlx5_flow_tunnel *tunnel, const void *x)
8918 {
8919         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8920
8921         RTE_SET_USED(dev);
8922         return tunnel->tunnel_id == ctx->tunnel_id;
8923 }
8924
8925 static void
8926 find_tunnel_id_hit(struct rte_eth_dev *dev,
8927                    struct mlx5_flow_tunnel *tunnel, void *x)
8928 {
8929         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8930         RTE_SET_USED(dev);
8931         ctx->tunnel = tunnel;
8932 }
8933
8934 static struct mlx5_flow_tunnel *
8935 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8936 {
8937         struct tunnel_db_find_tunnel_id_ctx ctx = {
8938                 .tunnel_id = id,
8939         };
8940
8941         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8942                                       find_tunnel_id_hit, NULL, &ctx, true);
8943
8944         return ctx.tunnel;
8945 }
8946
8947 static struct mlx5_flow_tunnel *
8948 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8949                           const struct rte_flow_tunnel *app_tunnel)
8950 {
8951         struct mlx5_priv *priv = dev->data->dev_private;
8952         struct mlx5_indexed_pool *ipool;
8953         struct mlx5_flow_tunnel *tunnel;
8954         uint32_t id;
8955
8956         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8957         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8958         if (!tunnel)
8959                 return NULL;
8960         if (id >= MLX5_MAX_TUNNELS) {
8961                 mlx5_ipool_free(ipool, id);
8962                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8963                 return NULL;
8964         }
8965         tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
8966                                            priv->sh,
8967                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8968                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8969                                            mlx5_flow_tunnel_grp2tbl_remove_cb,
8970                                            mlx5_flow_tunnel_grp2tbl_clone_cb,
8971                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
8972         if (!tunnel->groups) {
8973                 mlx5_ipool_free(ipool, id);
8974                 return NULL;
8975         }
8976         /* initiate new PMD tunnel */
8977         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8978         tunnel->tunnel_id = id;
8979         tunnel->action.type = (typeof(tunnel->action.type))
8980                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8981         tunnel->action.conf = tunnel;
8982         tunnel->item.type = (typeof(tunnel->item.type))
8983                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8984         tunnel->item.spec = tunnel;
8985         tunnel->item.last = NULL;
8986         tunnel->item.mask = NULL;
8987
8988         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8989                 dev->data->port_id, tunnel->tunnel_id);
8990
8991         return tunnel;
8992 }
8993
8994 struct tunnel_db_get_tunnel_ctx {
8995         const struct rte_flow_tunnel *app_tunnel;
8996         struct mlx5_flow_tunnel *tunnel;
8997 };
8998
8999 static bool get_tunnel_match(struct rte_eth_dev *dev,
9000                              struct mlx5_flow_tunnel *tunnel, const void *x)
9001 {
9002         const struct tunnel_db_get_tunnel_ctx *ctx = x;
9003
9004         RTE_SET_USED(dev);
9005         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
9006                        sizeof(*ctx->app_tunnel));
9007 }
9008
9009 static void get_tunnel_hit(struct rte_eth_dev *dev,
9010                            struct mlx5_flow_tunnel *tunnel, void *x)
9011 {
9012         /* called under tunnel spinlock protection */
9013         struct tunnel_db_get_tunnel_ctx *ctx = x;
9014
9015         RTE_SET_USED(dev);
9016         tunnel->refctn++;
9017         ctx->tunnel = tunnel;
9018 }
9019
9020 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
9021 {
9022         /* called under tunnel spinlock protection */
9023         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
9024         struct tunnel_db_get_tunnel_ctx *ctx = x;
9025
9026         rte_spinlock_unlock(&thub->sl);
9027         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
9028         rte_spinlock_lock(&thub->sl);
9029         if (ctx->tunnel) {
9030                 ctx->tunnel->refctn = 1;
9031                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
9032         }
9033 }
9034
9035
9036 static int
9037 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
9038                      const struct rte_flow_tunnel *app_tunnel,
9039                      struct mlx5_flow_tunnel **tunnel)
9040 {
9041         struct tunnel_db_get_tunnel_ctx ctx = {
9042                 .app_tunnel = app_tunnel,
9043         };
9044
9045         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
9046                                       get_tunnel_miss, &ctx, true);
9047         *tunnel = ctx.tunnel;
9048         return ctx.tunnel ? 0 : -ENOMEM;
9049 }
9050
9051 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
9052 {
9053         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
9054
9055         if (!thub)
9056                 return;
9057         if (!LIST_EMPTY(&thub->tunnels))
9058                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
9059         mlx5_hlist_destroy(thub->groups);
9060         mlx5_free(thub);
9061 }
9062
9063 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
9064 {
9065         int err;
9066         struct mlx5_flow_tunnel_hub *thub;
9067
9068         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
9069                            0, SOCKET_ID_ANY);
9070         if (!thub)
9071                 return -ENOMEM;
9072         LIST_INIT(&thub->tunnels);
9073         rte_spinlock_init(&thub->sl);
9074         thub->groups = mlx5_hlist_create("flow groups", 64,
9075                                          false, true, sh,
9076                                          mlx5_flow_tunnel_grp2tbl_create_cb,
9077                                          mlx5_flow_tunnel_grp2tbl_match_cb,
9078                                          mlx5_flow_tunnel_grp2tbl_remove_cb,
9079                                          mlx5_flow_tunnel_grp2tbl_clone_cb,
9080                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
9081         if (!thub->groups) {
9082                 err = -rte_errno;
9083                 goto err;
9084         }
9085         sh->tunnel_hub = thub;
9086
9087         return 0;
9088
9089 err:
9090         if (thub->groups)
9091                 mlx5_hlist_destroy(thub->groups);
9092         if (thub)
9093                 mlx5_free(thub);
9094         return err;
9095 }
9096
9097 static inline bool
9098 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
9099                           struct rte_flow_tunnel *tunnel,
9100                           const char *err_msg)
9101 {
9102         err_msg = NULL;
9103         if (!is_tunnel_offload_active(dev)) {
9104                 err_msg = "tunnel offload was not activated";
9105                 goto out;
9106         } else if (!tunnel) {
9107                 err_msg = "no application tunnel";
9108                 goto out;
9109         }
9110
9111         switch (tunnel->type) {
9112         default:
9113                 err_msg = "unsupported tunnel type";
9114                 goto out;
9115         case RTE_FLOW_ITEM_TYPE_VXLAN:
9116                 break;
9117         }
9118
9119 out:
9120         return !err_msg;
9121 }
9122
9123 static int
9124 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
9125                     struct rte_flow_tunnel *app_tunnel,
9126                     struct rte_flow_action **actions,
9127                     uint32_t *num_of_actions,
9128                     struct rte_flow_error *error)
9129 {
9130         int ret;
9131         struct mlx5_flow_tunnel *tunnel;
9132         const char *err_msg = NULL;
9133         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9134
9135         if (!verdict)
9136                 return rte_flow_error_set(error, EINVAL,
9137                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9138                                           err_msg);
9139         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9140         if (ret < 0) {
9141                 return rte_flow_error_set(error, ret,
9142                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9143                                           "failed to initialize pmd tunnel");
9144         }
9145         *actions = &tunnel->action;
9146         *num_of_actions = 1;
9147         return 0;
9148 }
9149
9150 static int
9151 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
9152                        struct rte_flow_tunnel *app_tunnel,
9153                        struct rte_flow_item **items,
9154                        uint32_t *num_of_items,
9155                        struct rte_flow_error *error)
9156 {
9157         int ret;
9158         struct mlx5_flow_tunnel *tunnel;
9159         const char *err_msg = NULL;
9160         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9161
9162         if (!verdict)
9163                 return rte_flow_error_set(error, EINVAL,
9164                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9165                                           err_msg);
9166         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9167         if (ret < 0) {
9168                 return rte_flow_error_set(error, ret,
9169                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9170                                           "failed to initialize pmd tunnel");
9171         }
9172         *items = &tunnel->item;
9173         *num_of_items = 1;
9174         return 0;
9175 }
9176
9177 struct tunnel_db_element_release_ctx {
9178         struct rte_flow_item *items;
9179         struct rte_flow_action *actions;
9180         uint32_t num_elements;
9181         struct rte_flow_error *error;
9182         int ret;
9183 };
9184
9185 static bool
9186 tunnel_element_release_match(struct rte_eth_dev *dev,
9187                              struct mlx5_flow_tunnel *tunnel, const void *x)
9188 {
9189         const struct tunnel_db_element_release_ctx *ctx = x;
9190
9191         RTE_SET_USED(dev);
9192         if (ctx->num_elements != 1)
9193                 return false;
9194         else if (ctx->items)
9195                 return ctx->items == &tunnel->item;
9196         else if (ctx->actions)
9197                 return ctx->actions == &tunnel->action;
9198
9199         return false;
9200 }
9201
9202 static void
9203 tunnel_element_release_hit(struct rte_eth_dev *dev,
9204                            struct mlx5_flow_tunnel *tunnel, void *x)
9205 {
9206         struct tunnel_db_element_release_ctx *ctx = x;
9207         ctx->ret = 0;
9208         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
9209                 mlx5_flow_tunnel_free(dev, tunnel);
9210 }
9211
9212 static void
9213 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
9214 {
9215         struct tunnel_db_element_release_ctx *ctx = x;
9216         RTE_SET_USED(dev);
9217         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
9218                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9219                                       "invalid argument");
9220 }
9221
9222 static int
9223 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
9224                        struct rte_flow_item *pmd_items,
9225                        uint32_t num_items, struct rte_flow_error *err)
9226 {
9227         struct tunnel_db_element_release_ctx ctx = {
9228                 .items = pmd_items,
9229                 .actions = NULL,
9230                 .num_elements = num_items,
9231                 .error = err,
9232         };
9233
9234         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9235                                       tunnel_element_release_hit,
9236                                       tunnel_element_release_miss, &ctx, false);
9237
9238         return ctx.ret;
9239 }
9240
9241 static int
9242 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
9243                          struct rte_flow_action *pmd_actions,
9244                          uint32_t num_actions, struct rte_flow_error *err)
9245 {
9246         struct tunnel_db_element_release_ctx ctx = {
9247                 .items = NULL,
9248                 .actions = pmd_actions,
9249                 .num_elements = num_actions,
9250                 .error = err,
9251         };
9252
9253         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9254                                       tunnel_element_release_hit,
9255                                       tunnel_element_release_miss, &ctx, false);
9256
9257         return ctx.ret;
9258 }
9259
9260 static int
9261 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
9262                                   struct rte_mbuf *m,
9263                                   struct rte_flow_restore_info *info,
9264                                   struct rte_flow_error *err)
9265 {
9266         uint64_t ol_flags = m->ol_flags;
9267         const struct mlx5_flow_tbl_data_entry *tble;
9268         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
9269
9270         if (!is_tunnel_offload_active(dev)) {
9271                 info->flags = 0;
9272                 return 0;
9273         }
9274
9275         if ((ol_flags & mask) != mask)
9276                 goto err;
9277         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
9278         if (!tble) {
9279                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
9280                         dev->data->port_id, m->hash.fdir.hi);
9281                 goto err;
9282         }
9283         MLX5_ASSERT(tble->tunnel);
9284         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
9285         info->group_id = tble->group_id;
9286         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
9287                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
9288                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
9289
9290         return 0;
9291
9292 err:
9293         return rte_flow_error_set(err, EINVAL,
9294                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9295                                   "failed to get restore info");
9296 }
9297
9298 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
9299 static int
9300 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
9301                            __rte_unused struct rte_flow_tunnel *app_tunnel,
9302                            __rte_unused struct rte_flow_action **actions,
9303                            __rte_unused uint32_t *num_of_actions,
9304                            __rte_unused struct rte_flow_error *error)
9305 {
9306         return -ENOTSUP;
9307 }
9308
9309 static int
9310 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
9311                        __rte_unused struct rte_flow_tunnel *app_tunnel,
9312                        __rte_unused struct rte_flow_item **items,
9313                        __rte_unused uint32_t *num_of_items,
9314                        __rte_unused struct rte_flow_error *error)
9315 {
9316         return -ENOTSUP;
9317 }
9318
9319 static int
9320 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
9321                               __rte_unused struct rte_flow_item *pmd_items,
9322                               __rte_unused uint32_t num_items,
9323                               __rte_unused struct rte_flow_error *err)
9324 {
9325         return -ENOTSUP;
9326 }
9327
9328 static int
9329 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
9330                                 __rte_unused struct rte_flow_action *pmd_action,
9331                                 __rte_unused uint32_t num_actions,
9332                                 __rte_unused struct rte_flow_error *err)
9333 {
9334         return -ENOTSUP;
9335 }
9336
9337 static int
9338 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
9339                                   __rte_unused struct rte_mbuf *m,
9340                                   __rte_unused struct rte_flow_restore_info *i,
9341                                   __rte_unused struct rte_flow_error *err)
9342 {
9343         return -ENOTSUP;
9344 }
9345
9346 static int
9347 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
9348                              __rte_unused struct rte_flow *flow,
9349                              __rte_unused const struct rte_flow_attr *attr,
9350                              __rte_unused const struct rte_flow_action *actions,
9351                              __rte_unused uint32_t flow_idx,
9352                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
9353                              __rte_unused struct tunnel_default_miss_ctx *ctx,
9354                              __rte_unused struct rte_flow_error *error)
9355 {
9356         return -ENOTSUP;
9357 }
9358
9359 static struct mlx5_flow_tunnel *
9360 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
9361                     __rte_unused uint32_t id)
9362 {
9363         return NULL;
9364 }
9365
9366 static void
9367 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
9368                       __rte_unused struct mlx5_flow_tunnel *tunnel)
9369 {
9370 }
9371
9372 static uint32_t
9373 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
9374                                 __rte_unused const struct mlx5_flow_tunnel *t,
9375                                 __rte_unused uint32_t group,
9376                                 __rte_unused uint32_t *table,
9377                                 struct rte_flow_error *error)
9378 {
9379         return rte_flow_error_set(error, ENOTSUP,
9380                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9381                                   "tunnel offload requires DV support");
9382 }
9383
9384 void
9385 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
9386                         __rte_unused  uint16_t port_id)
9387 {
9388 }
9389 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
9390
9391 static void
9392 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
9393 {
9394         int ret;
9395         struct rte_flow_error error;
9396
9397         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9398                 char *item_name;
9399                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
9400                                     sizeof(item_name),
9401                                     (void *)(uintptr_t)item->type, &error);
9402                 if (ret > 0)
9403                         printf("%s ", item_name);
9404                 else
9405                         printf("%d\n", (int)item->type);
9406         }
9407         printf("END\n");
9408 }