drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint8_t optional;
 104         /**< optional expand field. Default 0 to expand, 1 not go deeper. */
 105 };
 106
 107 /** Object returned by mlx5_flow_expand_rss(). */
 108 struct mlx5_flow_expand_rss {
 109         uint32_t entries;
 110         /**< Number of entries @p patterns and @p priorities. */
 111         struct {
 112                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 113                 uint32_t priority; /**< Priority offset for each expansion. */
 114         } entry[];
 115 };
 116
 117 static enum rte_flow_item_type
 118 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 119 {
 120         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 121         uint16_t ether_type = 0;
 122         uint16_t ether_type_m;
 123         uint8_t ip_next_proto = 0;
 124         uint8_t ip_next_proto_m;
 125
 126         if (item == NULL || item->spec == NULL)
 127                 return ret;
 128         switch (item->type) {
 129         case RTE_FLOW_ITEM_TYPE_ETH:
 130                 if (item->mask)
 131                         ether_type_m = ((const struct rte_flow_item_eth *)
 132                                                 (item->mask))->type;
 133                 else
 134                         ether_type_m = rte_flow_item_eth_mask.type;
 135                 if (ether_type_m != RTE_BE16(0xFFFF))
 136                         break;
 137                 ether_type = ((const struct rte_flow_item_eth *)
 138                                 (item->spec))->type;
 139                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 140                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 141                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 142                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 143                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 144                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 145                 else
 146                         ret = RTE_FLOW_ITEM_TYPE_END;
 147                 break;
 148         case RTE_FLOW_ITEM_TYPE_VLAN:
 149                 if (item->mask)
 150                         ether_type_m = ((const struct rte_flow_item_vlan *)
 151                                                 (item->mask))->inner_type;
 152                 else
 153                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 154                 if (ether_type_m != RTE_BE16(0xFFFF))
 155                         break;
 156                 ether_type = ((const struct rte_flow_item_vlan *)
 157                                 (item->spec))->inner_type;
 158                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 159                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 160                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 161                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 162                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 163                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 164                 else
 165                         ret = RTE_FLOW_ITEM_TYPE_END;
 166                 break;
 167         case RTE_FLOW_ITEM_TYPE_IPV4:
 168                 if (item->mask)
 169                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 170                                         (item->mask))->hdr.next_proto_id;
 171                 else
 172                         ip_next_proto_m =
 173                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 174                 if (ip_next_proto_m != 0xFF)
 175                         break;
 176                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 177                                 (item->spec))->hdr.next_proto_id;
 178                 if (ip_next_proto == IPPROTO_UDP)
 179                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 180                 else if (ip_next_proto == IPPROTO_TCP)
 181                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 182                 else if (ip_next_proto == IPPROTO_IP)
 183                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 else if (ip_next_proto == IPPROTO_IPV6)
 185                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 186                 else
 187                         ret = RTE_FLOW_ITEM_TYPE_END;
 188                 break;
 189         case RTE_FLOW_ITEM_TYPE_IPV6:
 190                 if (item->mask)
 191                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 192                                                 (item->mask))->hdr.proto;
 193                 else
 194                         ip_next_proto_m =
 195                                 rte_flow_item_ipv6_mask.hdr.proto;
 196                 if (ip_next_proto_m != 0xFF)
 197                         break;
 198                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 199                                 (item->spec))->hdr.proto;
 200                 if (ip_next_proto == IPPROTO_UDP)
 201                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 202                 else if (ip_next_proto == IPPROTO_TCP)
 203                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 204                 else if (ip_next_proto == IPPROTO_IP)
 205                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 206                 else if (ip_next_proto == IPPROTO_IPV6)
 207                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 208                 else
 209                         ret = RTE_FLOW_ITEM_TYPE_END;
 210                 break;
 211         default:
 212                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 213                 break;
 214         }
 215         return ret;
 216 }
 217
 218 #define MLX5_RSS_EXP_ELT_N 16
 219
 220 /**
 221  * Expand RSS flows into several possible flows according to the RSS hash
 222  * fields requested and the driver capabilities.
 223  *
 224  * @param[out] buf
 225  *   Buffer to store the result expansion.
 226  * @param[in] size
 227  *   Buffer size in bytes. If 0, @p buf can be NULL.
 228  * @param[in] pattern
 229  *   User flow pattern.
 230  * @param[in] types
 231  *   RSS types to expand (see ETH_RSS_* definitions).
 232  * @param[in] graph
 233  *   Input graph to expand @p pattern according to @p types.
 234  * @param[in] graph_root_index
 235  *   Index of root node in @p graph, typically 0.
 236  *
 237  * @return
 238  *   A positive value representing the size of @p buf in bytes regardless of
 239  *   @p size on success, a negative errno value otherwise and rte_errno is
 240  *   set, the following errors are defined:
 241  *
 242  *   -E2BIG: graph-depth @p graph is too deep.
 243  */
 244 static int
 245 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 246                      const struct rte_flow_item *pattern, uint64_t types,
 247                      const struct mlx5_flow_expand_node graph[],
 248                      int graph_root_index)
 249 {
 250         const struct rte_flow_item *item;
 251         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 252         const int *next_node;
 253         const int *stack[MLX5_RSS_EXP_ELT_N];
 254         int stack_pos = 0;
 255         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 256         unsigned int i;
 257         size_t lsize;
 258         size_t user_pattern_size = 0;
 259         void *addr = NULL;
 260         const struct mlx5_flow_expand_node *next = NULL;
 261         struct rte_flow_item missed_item;
 262         int missed = 0;
 263         int elt = 0;
 264         const struct rte_flow_item *last_item = NULL;
 265
 266         memset(&missed_item, 0, sizeof(missed_item));
 267         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 268                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 269         if (lsize <= size) {
 270                 buf->entry[0].priority = 0;
 271                 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 272                 buf->entries = 0;
 273                 addr = buf->entry[0].pattern;
 274         }
 275         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 277                         last_item = item;
 278                 for (i = 0; node->next && node->next[i]; ++i) {
 279                         next = &graph[node->next[i]];
 280                         if (next->type == item->type)
 281                                 break;
 282                 }
 283                 if (next)
 284                         node = next;
 285                 user_pattern_size += sizeof(*item);
 286         }
 287         user_pattern_size += sizeof(*item); /* Handle END item. */
 288         lsize += user_pattern_size;
 289         /* Copy the user pattern in the first entry of the buffer. */
 290         if (lsize <= size) {
 291                 rte_memcpy(addr, pattern, user_pattern_size);
 292                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 293                 buf->entries = 1;
 294         }
 295         /* Start expanding. */
 296         memset(flow_items, 0, sizeof(flow_items));
 297         user_pattern_size -= sizeof(*item);
 298         /*
 299          * Check if the last valid item has spec set, need complete pattern,
 300          * and the pattern can be used for expansion.
 301          */
 302         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 303         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 304                 /* Item type END indicates expansion is not required. */
 305                 return lsize;
 306         }
 307         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 308                 next = NULL;
 309                 missed = 1;
 310                 for (i = 0; node->next && node->next[i]; ++i) {
 311                         next = &graph[node->next[i]];
 312                         if (next->type == missed_item.type) {
 313                                 flow_items[0].type = missed_item.type;
 314                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 315                                 break;
 316                         }
 317                         next = NULL;
 318                 }
 319         }
 320         if (next && missed) {
 321                 elt = 2; /* missed item + item end. */
 322                 node = next;
 323                 lsize += elt * sizeof(*item) + user_pattern_size;
 324                 if ((node->rss_types & types) && lsize <= size) {
 325                         buf->entry[buf->entries].priority = 1;
 326                         buf->entry[buf->entries].pattern = addr;
 327                         buf->entries++;
 328                         rte_memcpy(addr, buf->entry[0].pattern,
 329                                    user_pattern_size);
 330                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 331                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 332                         addr = (void *)(((uintptr_t)addr) +
 333                                         elt * sizeof(*item));
 334                 }
 335         }
 336         memset(flow_items, 0, sizeof(flow_items));
 337         next_node = node->next;
 338         stack[stack_pos] = next_node;
 339         node = next_node ? &graph[*next_node] : NULL;
 340         while (node) {
 341                 flow_items[stack_pos].type = node->type;
 342                 if (node->rss_types & types) {
 343                         /*
 344                          * compute the number of items to copy from the
 345                          * expansion and copy it.
 346                          * When the stack_pos is 0, there are 1 element in it,
 347                          * plus the addition END item.
 348                          */
 349                         elt = stack_pos + 2;
 350                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 351                         lsize += elt * sizeof(*item) + user_pattern_size;
 352                         if (lsize <= size) {
 353                                 size_t n = elt * sizeof(*item);
 354
 355                                 buf->entry[buf->entries].priority =
 356                                         stack_pos + 1 + missed;
 357                                 buf->entry[buf->entries].pattern = addr;
 358                                 buf->entries++;
 359                                 rte_memcpy(addr, buf->entry[0].pattern,
 360                                            user_pattern_size);
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                                 user_pattern_size);
 363                                 rte_memcpy(addr, &missed_item,
 364                                            missed * sizeof(*item));
 365                                 addr = (void *)(((uintptr_t)addr) +
 366                                         missed * sizeof(*item));
 367                                 rte_memcpy(addr, flow_items, n);
 368                                 addr = (void *)(((uintptr_t)addr) + n);
 369                         }
 370                 }
 371                 /* Go deeper. */
 372                 if (!node->optional && node->next) {
 373                         next_node = node->next;
 374                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 375                                 rte_errno = E2BIG;
 376                                 return -rte_errno;
 377                         }
 378                         stack[stack_pos] = next_node;
 379                 } else if (*(next_node + 1)) {
 380                         /* Follow up with the next possibility. */
 381                         ++next_node;
 382                 } else {
 383                         /* Move to the next path. */
 384                         if (stack_pos)
 385                                 next_node = stack[--stack_pos];
 386                         next_node++;
 387                         stack[stack_pos] = next_node;
 388                 }
 389                 node = *next_node ? &graph[*next_node] : NULL;
 390         };
 391         return lsize;
 392 }
 393
 394 enum mlx5_expansion {
 395         MLX5_EXPANSION_ROOT,
 396         MLX5_EXPANSION_ROOT_OUTER,
 397         MLX5_EXPANSION_ROOT_ETH_VLAN,
 398         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 399         MLX5_EXPANSION_OUTER_ETH,
 400         MLX5_EXPANSION_OUTER_ETH_VLAN,
 401         MLX5_EXPANSION_OUTER_VLAN,
 402         MLX5_EXPANSION_OUTER_IPV4,
 403         MLX5_EXPANSION_OUTER_IPV4_UDP,
 404         MLX5_EXPANSION_OUTER_IPV4_TCP,
 405         MLX5_EXPANSION_OUTER_IPV6,
 406         MLX5_EXPANSION_OUTER_IPV6_UDP,
 407         MLX5_EXPANSION_OUTER_IPV6_TCP,
 408         MLX5_EXPANSION_VXLAN,
 409         MLX5_EXPANSION_VXLAN_GPE,
 410         MLX5_EXPANSION_GRE,
 411         MLX5_EXPANSION_GRE_KEY,
 412         MLX5_EXPANSION_MPLS,
 413         MLX5_EXPANSION_ETH,
 414         MLX5_EXPANSION_ETH_VLAN,
 415         MLX5_EXPANSION_VLAN,
 416         MLX5_EXPANSION_IPV4,
 417         MLX5_EXPANSION_IPV4_UDP,
 418         MLX5_EXPANSION_IPV4_TCP,
 419         MLX5_EXPANSION_IPV6,
 420         MLX5_EXPANSION_IPV6_UDP,
 421         MLX5_EXPANSION_IPV6_TCP,
 422 };
 423
 424 /** Supported expansion of items. */
 425 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 426         [MLX5_EXPANSION_ROOT] = {
 427                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 428                                                   MLX5_EXPANSION_IPV4,
 429                                                   MLX5_EXPANSION_IPV6),
 430                 .type = RTE_FLOW_ITEM_TYPE_END,
 431         },
 432         [MLX5_EXPANSION_ROOT_OUTER] = {
 433                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 434                                                   MLX5_EXPANSION_OUTER_IPV4,
 435                                                   MLX5_EXPANSION_OUTER_IPV6),
 436                 .type = RTE_FLOW_ITEM_TYPE_END,
 437         },
 438         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 439                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 440                 .type = RTE_FLOW_ITEM_TYPE_END,
 441         },
 442         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 443                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 444                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 445                 .type = RTE_FLOW_ITEM_TYPE_END,
 446         },
 447         [MLX5_EXPANSION_OUTER_ETH] = {
 448                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 449                                                   MLX5_EXPANSION_OUTER_IPV6,
 450                                                   MLX5_EXPANSION_MPLS),
 451                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 452                 .rss_types = 0,
 453         },
 454         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 455                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 457                 .rss_types = 0,
 458         },
 459         [MLX5_EXPANSION_OUTER_VLAN] = {
 460                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 461                                                   MLX5_EXPANSION_OUTER_IPV6),
 462                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 463         },
 464         [MLX5_EXPANSION_OUTER_IPV4] = {
 465                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 466                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 467                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 468                          MLX5_EXPANSION_GRE,
 469                          MLX5_EXPANSION_IPV4,
 470                          MLX5_EXPANSION_IPV6),
 471                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 472                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 473                         ETH_RSS_NONFRAG_IPV4_OTHER,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 477                                                   MLX5_EXPANSION_VXLAN_GPE),
 478                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 479                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 480         },
 481         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 482                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 483                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 484         },
 485         [MLX5_EXPANSION_OUTER_IPV6] = {
 486                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 487                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 488                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 489                          MLX5_EXPANSION_IPV4,
 490                          MLX5_EXPANSION_IPV6,
 491                          MLX5_EXPANSION_GRE),
 492                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 493                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 494                         ETH_RSS_NONFRAG_IPV6_OTHER,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 498                                                   MLX5_EXPANSION_VXLAN_GPE),
 499                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 500                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 501         },
 502         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 503                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 504                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 505         },
 506         [MLX5_EXPANSION_VXLAN] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 508                                                   MLX5_EXPANSION_IPV4,
 509                                                   MLX5_EXPANSION_IPV6),
 510                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 511         },
 512         [MLX5_EXPANSION_VXLAN_GPE] = {
 513                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 514                                                   MLX5_EXPANSION_IPV4,
 515                                                   MLX5_EXPANSION_IPV6),
 516                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 517         },
 518         [MLX5_EXPANSION_GRE] = {
 519                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 520                                                   MLX5_EXPANSION_IPV6,
 521                                                   MLX5_EXPANSION_GRE_KEY),
 522                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 523         },
 524         [MLX5_EXPANSION_GRE_KEY] = {
 525                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 526                                                   MLX5_EXPANSION_IPV6),
 527                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 528                 .optional = 1,
 529         },
 530         [MLX5_EXPANSION_MPLS] = {
 531                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 532                                                   MLX5_EXPANSION_IPV6),
 533                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 534         },
 535         [MLX5_EXPANSION_ETH] = {
 536                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 537                                                   MLX5_EXPANSION_IPV6),
 538                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 539         },
 540         [MLX5_EXPANSION_ETH_VLAN] = {
 541                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 542                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 543         },
 544         [MLX5_EXPANSION_VLAN] = {
 545                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 546                                                   MLX5_EXPANSION_IPV6),
 547                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 548         },
 549         [MLX5_EXPANSION_IPV4] = {
 550                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 551                                                   MLX5_EXPANSION_IPV4_TCP),
 552                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 553                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 554                         ETH_RSS_NONFRAG_IPV4_OTHER,
 555         },
 556         [MLX5_EXPANSION_IPV4_UDP] = {
 557                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 558                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 559         },
 560         [MLX5_EXPANSION_IPV4_TCP] = {
 561                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 562                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 563         },
 564         [MLX5_EXPANSION_IPV6] = {
 565                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 566                                                   MLX5_EXPANSION_IPV6_TCP),
 567                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 568                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 569                         ETH_RSS_NONFRAG_IPV6_OTHER,
 570         },
 571         [MLX5_EXPANSION_IPV6_UDP] = {
 572                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 573                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 574         },
 575         [MLX5_EXPANSION_IPV6_TCP] = {
 576                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 577                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 578         },
 579 };
 580
 581 static struct rte_flow_action_handle *
 582 mlx5_action_handle_create(struct rte_eth_dev *dev,
 583                           const struct rte_flow_indir_action_conf *conf,
 584                           const struct rte_flow_action *action,
 585                           struct rte_flow_error *error);
 586 static int mlx5_action_handle_destroy
 587                                 (struct rte_eth_dev *dev,
 588                                  struct rte_flow_action_handle *handle,
 589                                  struct rte_flow_error *error);
 590 static int mlx5_action_handle_update
 591                                 (struct rte_eth_dev *dev,
 592                                  struct rte_flow_action_handle *handle,
 593                                  const void *update,
 594                                  struct rte_flow_error *error);
 595 static int mlx5_action_handle_query
 596                                 (struct rte_eth_dev *dev,
 597                                  const struct rte_flow_action_handle *handle,
 598                                  void *data,
 599                                  struct rte_flow_error *error);
 600 static int
 601 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 602                     struct rte_flow_tunnel *app_tunnel,
 603                     struct rte_flow_action **actions,
 604                     uint32_t *num_of_actions,
 605                     struct rte_flow_error *error);
 606 static int
 607 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 608                        struct rte_flow_tunnel *app_tunnel,
 609                        struct rte_flow_item **items,
 610                        uint32_t *num_of_items,
 611                        struct rte_flow_error *error);
 612 static int
 613 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 614                               struct rte_flow_item *pmd_items,
 615                               uint32_t num_items, struct rte_flow_error *err);
 616 static int
 617 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 618                                 struct rte_flow_action *pmd_actions,
 619                                 uint32_t num_actions,
 620                                 struct rte_flow_error *err);
 621 static int
 622 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 623                                   struct rte_mbuf *m,
 624                                   struct rte_flow_restore_info *info,
 625                                   struct rte_flow_error *err);
 626
 627 static const struct rte_flow_ops mlx5_flow_ops = {
 628         .validate = mlx5_flow_validate,
 629         .create = mlx5_flow_create,
 630         .destroy = mlx5_flow_destroy,
 631         .flush = mlx5_flow_flush,
 632         .isolate = mlx5_flow_isolate,
 633         .query = mlx5_flow_query,
 634         .dev_dump = mlx5_flow_dev_dump,
 635         .get_aged_flows = mlx5_flow_get_aged_flows,
 636         .action_handle_create = mlx5_action_handle_create,
 637         .action_handle_destroy = mlx5_action_handle_destroy,
 638         .action_handle_update = mlx5_action_handle_update,
 639         .action_handle_query = mlx5_action_handle_query,
 640         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 641         .tunnel_match = mlx5_flow_tunnel_match,
 642         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 643         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 644         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 645 };
 646
 647 /* Tunnel information. */
 648 struct mlx5_flow_tunnel_info {
 649         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 650         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 651 };
 652
 653 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 654         {
 655                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 656                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 657         },
 658         {
 659                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 660                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 661         },
 662         {
 663                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 664                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 665         },
 666         {
 667                 .tunnel = MLX5_FLOW_LAYER_GRE,
 668                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 669         },
 670         {
 671                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 672                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 673         },
 674         {
 675                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 676                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 677         },
 678         {
 679                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 680                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 681         },
 682         {
 683                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 684                 .ptype = RTE_PTYPE_TUNNEL_IP,
 685         },
 686         {
 687                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 688                 .ptype = RTE_PTYPE_TUNNEL_IP,
 689         },
 690         {
 691                 .tunnel = MLX5_FLOW_LAYER_GTP,
 692                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 693         },
 694 };
 695
 696
 697
 698 /**
 699  * Translate tag ID to register.
 700  *
 701  * @param[in] dev
 702  *   Pointer to the Ethernet device structure.
 703  * @param[in] feature
 704  *   The feature that request the register.
 705  * @param[in] id
 706  *   The request register ID.
 707  * @param[out] error
 708  *   Error description in case of any.
 709  *
 710  * @return
 711  *   The request register on success, a negative errno
 712  *   value otherwise and rte_errno is set.
 713  */
 714 int
 715 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 716                      enum mlx5_feature_name feature,
 717                      uint32_t id,
 718                      struct rte_flow_error *error)
 719 {
 720         struct mlx5_priv *priv = dev->data->dev_private;
 721         struct mlx5_dev_config *config = &priv->config;
 722         enum modify_reg start_reg;
 723         bool skip_mtr_reg = false;
 724
 725         switch (feature) {
 726         case MLX5_HAIRPIN_RX:
 727                 return REG_B;
 728         case MLX5_HAIRPIN_TX:
 729                 return REG_A;
 730         case MLX5_METADATA_RX:
 731                 switch (config->dv_xmeta_en) {
 732                 case MLX5_XMETA_MODE_LEGACY:
 733                         return REG_B;
 734                 case MLX5_XMETA_MODE_META16:
 735                         return REG_C_0;
 736                 case MLX5_XMETA_MODE_META32:
 737                         return REG_C_1;
 738                 }
 739                 break;
 740         case MLX5_METADATA_TX:
 741                 return REG_A;
 742         case MLX5_METADATA_FDB:
 743                 switch (config->dv_xmeta_en) {
 744                 case MLX5_XMETA_MODE_LEGACY:
 745                         return REG_NON;
 746                 case MLX5_XMETA_MODE_META16:
 747                         return REG_C_0;
 748                 case MLX5_XMETA_MODE_META32:
 749                         return REG_C_1;
 750                 }
 751                 break;
 752         case MLX5_FLOW_MARK:
 753                 switch (config->dv_xmeta_en) {
 754                 case MLX5_XMETA_MODE_LEGACY:
 755                         return REG_NON;
 756                 case MLX5_XMETA_MODE_META16:
 757                         return REG_C_1;
 758                 case MLX5_XMETA_MODE_META32:
 759                         return REG_C_0;
 760                 }
 761                 break;
 762         case MLX5_MTR_ID:
 763                 /*
 764                  * If meter color and meter id share one register, flow match
 765                  * should use the meter color register for match.
 766                  */
 767                 if (priv->mtr_reg_share)
 768                         return priv->mtr_color_reg;
 769                 else
 770                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 771                                REG_C_3;
 772         case MLX5_MTR_COLOR:
 773         case MLX5_ASO_FLOW_HIT:
 774         case MLX5_ASO_CONNTRACK:
 775                 /* All features use the same REG_C. */
 776                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 777                 return priv->mtr_color_reg;
 778         case MLX5_COPY_MARK:
 779                 /*
 780                  * Metadata COPY_MARK register using is in meter suffix sub
 781                  * flow while with meter. It's safe to share the same register.
 782                  */
 783                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 784         case MLX5_APP_TAG:
 785                 /*
 786                  * If meter is enable, it will engage the register for color
 787                  * match and flow match. If meter color match is not using the
 788                  * REG_C_2, need to skip the REG_C_x be used by meter color
 789                  * match.
 790                  * If meter is disable, free to use all available registers.
 791                  */
 792                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 793                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 794                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 795                 if (id > (uint32_t)(REG_C_7 - start_reg))
 796                         return rte_flow_error_set(error, EINVAL,
 797                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 798                                                   NULL, "invalid tag id");
 799                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 800                         return rte_flow_error_set(error, ENOTSUP,
 801                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 802                                                   NULL, "unsupported tag id");
 803                 /*
 804                  * This case means meter is using the REG_C_x great than 2.
 805                  * Take care not to conflict with meter color REG_C_x.
 806                  * If the available index REG_C_y >= REG_C_x, skip the
 807                  * color register.
 808                  */
 809                 if (skip_mtr_reg && config->flow_mreg_c
 810                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 811                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 812                                 return rte_flow_error_set(error, EINVAL,
 813                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 814                                                         NULL, "invalid tag id");
 815                         if (config->flow_mreg_c
 816                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 817                                 return config->flow_mreg_c
 818                                                [id + 1 + start_reg - REG_C_0];
 819                         return rte_flow_error_set(error, ENOTSUP,
 820                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 821                                                   NULL, "unsupported tag id");
 822                 }
 823                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 824         }
 825         MLX5_ASSERT(false);
 826         return rte_flow_error_set(error, EINVAL,
 827                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 828                                   NULL, "invalid feature name");
 829 }
 830
 831 /**
 832  * Check extensive flow metadata register support.
 833  *
 834  * @param dev
 835  *   Pointer to rte_eth_dev structure.
 836  *
 837  * @return
 838  *   True if device supports extensive flow metadata register, otherwise false.
 839  */
 840 bool
 841 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 842 {
 843         struct mlx5_priv *priv = dev->data->dev_private;
 844         struct mlx5_dev_config *config = &priv->config;
 845
 846         /*
 847          * Having available reg_c can be regarded inclusively as supporting
 848          * extensive flow metadata register, which could mean,
 849          * - metadata register copy action by modify header.
 850          * - 16 modify header actions is supported.
 851          * - reg_c's are preserved across different domain (FDB and NIC) on
 852          *   packet loopback by flow lookup miss.
 853          */
 854         return config->flow_mreg_c[2] != REG_NON;
 855 }
 856
 857 /**
 858  * Get the lowest priority.
 859  *
 860  * @param[in] dev
 861  *   Pointer to the Ethernet device structure.
 862  * @param[in] attributes
 863  *   Pointer to device flow rule attributes.
 864  *
 865  * @return
 866  *   The value of lowest priority of flow.
 867  */
 868 uint32_t
 869 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 870                           const struct rte_flow_attr *attr)
 871 {
 872         struct mlx5_priv *priv = dev->data->dev_private;
 873
 874         if (!attr->group && !attr->transfer)
 875                 return priv->config.flow_prio - 2;
 876         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 877 }
 878
 879 /**
 880  * Calculate matcher priority of the flow.
 881  *
 882  * @param[in] dev
 883  *   Pointer to the Ethernet device structure.
 884  * @param[in] attr
 885  *   Pointer to device flow rule attributes.
 886  * @param[in] subpriority
 887  *   The priority based on the items.
 888  * @return
 889  *   The matcher priority of the flow.
 890  */
 891 uint16_t
 892 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 893                           const struct rte_flow_attr *attr,
 894                           uint32_t subpriority)
 895 {
 896         uint16_t priority = (uint16_t)attr->priority;
 897         struct mlx5_priv *priv = dev->data->dev_private;
 898
 899         if (!attr->group && !attr->transfer) {
 900                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 901                         priority = priv->config.flow_prio - 1;
 902                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 903         }
 904         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 905                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 906         return priority * 3 + subpriority;
 907 }
 908
 909 /**
 910  * Verify the @p item specifications (spec, last, mask) are compatible with the
 911  * NIC capabilities.
 912  *
 913  * @param[in] item
 914  *   Item specification.
 915  * @param[in] mask
 916  *   @p item->mask or flow default bit-masks.
 917  * @param[in] nic_mask
 918  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 919  * @param[in] size
 920  *   Bit-masks size in bytes.
 921  * @param[in] range_accepted
 922  *   True if range of values is accepted for specific fields, false otherwise.
 923  * @param[out] error
 924  *   Pointer to error structure.
 925  *
 926  * @return
 927  *   0 on success, a negative errno value otherwise and rte_errno is set.
 928  */
 929 int
 930 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 931                           const uint8_t *mask,
 932                           const uint8_t *nic_mask,
 933                           unsigned int size,
 934                           bool range_accepted,
 935                           struct rte_flow_error *error)
 936 {
 937         unsigned int i;
 938
 939         MLX5_ASSERT(nic_mask);
 940         for (i = 0; i < size; ++i)
 941                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 942                         return rte_flow_error_set(error, ENOTSUP,
 943                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 944                                                   item,
 945                                                   "mask enables non supported"
 946                                                   " bits");
 947         if (!item->spec && (item->mask || item->last))
 948                 return rte_flow_error_set(error, EINVAL,
 949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 950                                           "mask/last without a spec is not"
 951                                           " supported");
 952         if (item->spec && item->last && !range_accepted) {
 953                 uint8_t spec[size];
 954                 uint8_t last[size];
 955                 unsigned int i;
 956                 int ret;
 957
 958                 for (i = 0; i < size; ++i) {
 959                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 960                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 961                 }
 962                 ret = memcmp(spec, last, size);
 963                 if (ret != 0)
 964                         return rte_flow_error_set(error, EINVAL,
 965                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 966                                                   item,
 967                                                   "range is not valid");
 968         }
 969         return 0;
 970 }
 971
 972 /**
 973  * Adjust the hash fields according to the @p flow information.
 974  *
 975  * @param[in] dev_flow.
 976  *   Pointer to the mlx5_flow.
 977  * @param[in] tunnel
 978  *   1 when the hash field is for a tunnel item.
 979  * @param[in] layer_types
 980  *   ETH_RSS_* types.
 981  * @param[in] hash_fields
 982  *   Item hash fields.
 983  *
 984  * @return
 985  *   The hash fields that should be used.
 986  */
 987 uint64_t
 988 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 989                             int tunnel __rte_unused, uint64_t layer_types,
 990                             uint64_t hash_fields)
 991 {
 992 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 993         int rss_request_inner = rss_desc->level >= 2;
 994
 995         /* Check RSS hash level for tunnel. */
 996         if (tunnel && rss_request_inner)
 997                 hash_fields |= IBV_RX_HASH_INNER;
 998         else if (tunnel || rss_request_inner)
 999                 return 0;
1000 #endif
1001         /* Check if requested layer matches RSS hash fields. */
1002         if (!(rss_desc->types & layer_types))
1003                 return 0;
1004         return hash_fields;
1005 }
1006
1007 /**
1008  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1009  * if several tunnel rules are used on this queue, the tunnel ptype will be
1010  * cleared.
1011  *
1012  * @param rxq_ctrl
1013  *   Rx queue to update.
1014  */
1015 static void
1016 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1017 {
1018         unsigned int i;
1019         uint32_t tunnel_ptype = 0;
1020
1021         /* Look up for the ptype to use. */
1022         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1023                 if (!rxq_ctrl->flow_tunnels_n[i])
1024                         continue;
1025                 if (!tunnel_ptype) {
1026                         tunnel_ptype = tunnels_info[i].ptype;
1027                 } else {
1028                         tunnel_ptype = 0;
1029                         break;
1030                 }
1031         }
1032         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1033 }
1034
1035 /**
1036  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1037  * flow.
1038  *
1039  * @param[in] dev
1040  *   Pointer to the Ethernet device structure.
1041  * @param[in] dev_handle
1042  *   Pointer to device flow handle structure.
1043  */
1044 void
1045 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1046                        struct mlx5_flow_handle *dev_handle)
1047 {
1048         struct mlx5_priv *priv = dev->data->dev_private;
1049         const int mark = dev_handle->mark;
1050         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1051         struct mlx5_ind_table_obj *ind_tbl = NULL;
1052         unsigned int i;
1053
1054         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1055                 struct mlx5_hrxq *hrxq;
1056
1057                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1058                               dev_handle->rix_hrxq);
1059                 if (hrxq)
1060                         ind_tbl = hrxq->ind_table;
1061         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1062                 struct mlx5_shared_action_rss *shared_rss;
1063
1064                 shared_rss = mlx5_ipool_get
1065                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1066                          dev_handle->rix_srss);
1067                 if (shared_rss)
1068                         ind_tbl = shared_rss->ind_tbl;
1069         }
1070         if (!ind_tbl)
1071                 return;
1072         for (i = 0; i != ind_tbl->queues_n; ++i) {
1073                 int idx = ind_tbl->queues[i];
1074                 struct mlx5_rxq_ctrl *rxq_ctrl =
1075                         container_of((*priv->rxqs)[idx],
1076                                      struct mlx5_rxq_ctrl, rxq);
1077
1078                 /*
1079                  * To support metadata register copy on Tx loopback,
1080                  * this must be always enabled (metadata may arive
1081                  * from other port - not from local flows only.
1082                  */
1083                 if (priv->config.dv_flow_en &&
1084                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1085                     mlx5_flow_ext_mreg_supported(dev)) {
1086                         rxq_ctrl->rxq.mark = 1;
1087                         rxq_ctrl->flow_mark_n = 1;
1088                 } else if (mark) {
1089                         rxq_ctrl->rxq.mark = 1;
1090                         rxq_ctrl->flow_mark_n++;
1091                 }
1092                 if (tunnel) {
1093                         unsigned int j;
1094
1095                         /* Increase the counter matching the flow. */
1096                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1097                                 if ((tunnels_info[j].tunnel &
1098                                      dev_handle->layers) ==
1099                                     tunnels_info[j].tunnel) {
1100                                         rxq_ctrl->flow_tunnels_n[j]++;
1101                                         break;
1102                                 }
1103                         }
1104                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1105                 }
1106         }
1107 }
1108
1109 /**
1110  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1111  *
1112  * @param[in] dev
1113  *   Pointer to the Ethernet device structure.
1114  * @param[in] flow
1115  *   Pointer to flow structure.
1116  */
1117 static void
1118 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1119 {
1120         struct mlx5_priv *priv = dev->data->dev_private;
1121         uint32_t handle_idx;
1122         struct mlx5_flow_handle *dev_handle;
1123
1124         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1125                        handle_idx, dev_handle, next)
1126                 flow_drv_rxq_flags_set(dev, dev_handle);
1127 }
1128
1129 /**
1130  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1131  * device flow if no other flow uses it with the same kind of request.
1132  *
1133  * @param dev
1134  *   Pointer to Ethernet device.
1135  * @param[in] dev_handle
1136  *   Pointer to the device flow handle structure.
1137  */
1138 static void
1139 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1140                         struct mlx5_flow_handle *dev_handle)
1141 {
1142         struct mlx5_priv *priv = dev->data->dev_private;
1143         const int mark = dev_handle->mark;
1144         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1145         struct mlx5_ind_table_obj *ind_tbl = NULL;
1146         unsigned int i;
1147
1148         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1149                 struct mlx5_hrxq *hrxq;
1150
1151                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1152                               dev_handle->rix_hrxq);
1153                 if (hrxq)
1154                         ind_tbl = hrxq->ind_table;
1155         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1156                 struct mlx5_shared_action_rss *shared_rss;
1157
1158                 shared_rss = mlx5_ipool_get
1159                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1160                          dev_handle->rix_srss);
1161                 if (shared_rss)
1162                         ind_tbl = shared_rss->ind_tbl;
1163         }
1164         if (!ind_tbl)
1165                 return;
1166         MLX5_ASSERT(dev->data->dev_started);
1167         for (i = 0; i != ind_tbl->queues_n; ++i) {
1168                 int idx = ind_tbl->queues[i];
1169                 struct mlx5_rxq_ctrl *rxq_ctrl =
1170                         container_of((*priv->rxqs)[idx],
1171                                      struct mlx5_rxq_ctrl, rxq);
1172
1173                 if (priv->config.dv_flow_en &&
1174                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1175                     mlx5_flow_ext_mreg_supported(dev)) {
1176                         rxq_ctrl->rxq.mark = 1;
1177                         rxq_ctrl->flow_mark_n = 1;
1178                 } else if (mark) {
1179                         rxq_ctrl->flow_mark_n--;
1180                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1181                 }
1182                 if (tunnel) {
1183                         unsigned int j;
1184
1185                         /* Decrease the counter matching the flow. */
1186                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1187                                 if ((tunnels_info[j].tunnel &
1188                                      dev_handle->layers) ==
1189                                     tunnels_info[j].tunnel) {
1190                                         rxq_ctrl->flow_tunnels_n[j]--;
1191                                         break;
1192                                 }
1193                         }
1194                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1195                 }
1196         }
1197 }
1198
1199 /**
1200  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1201  * @p flow if no other flow uses it with the same kind of request.
1202  *
1203  * @param dev
1204  *   Pointer to Ethernet device.
1205  * @param[in] flow
1206  *   Pointer to the flow.
1207  */
1208 static void
1209 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1210 {
1211         struct mlx5_priv *priv = dev->data->dev_private;
1212         uint32_t handle_idx;
1213         struct mlx5_flow_handle *dev_handle;
1214
1215         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1216                        handle_idx, dev_handle, next)
1217                 flow_drv_rxq_flags_trim(dev, dev_handle);
1218 }
1219
1220 /**
1221  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1222  *
1223  * @param dev
1224  *   Pointer to Ethernet device.
1225  */
1226 static void
1227 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1228 {
1229         struct mlx5_priv *priv = dev->data->dev_private;
1230         unsigned int i;
1231
1232         for (i = 0; i != priv->rxqs_n; ++i) {
1233                 struct mlx5_rxq_ctrl *rxq_ctrl;
1234                 unsigned int j;
1235
1236                 if (!(*priv->rxqs)[i])
1237                         continue;
1238                 rxq_ctrl = container_of((*priv->rxqs)[i],
1239                                         struct mlx5_rxq_ctrl, rxq);
1240                 rxq_ctrl->flow_mark_n = 0;
1241                 rxq_ctrl->rxq.mark = 0;
1242                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1243                         rxq_ctrl->flow_tunnels_n[j] = 0;
1244                 rxq_ctrl->rxq.tunnel = 0;
1245         }
1246 }
1247
1248 /**
1249  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1250  *
1251  * @param[in] dev
1252  *   Pointer to the Ethernet device structure.
1253  */
1254 void
1255 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1256 {
1257         struct mlx5_priv *priv = dev->data->dev_private;
1258         struct mlx5_rxq_data *data;
1259         unsigned int i;
1260
1261         for (i = 0; i != priv->rxqs_n; ++i) {
1262                 if (!(*priv->rxqs)[i])
1263                         continue;
1264                 data = (*priv->rxqs)[i];
1265                 if (!rte_flow_dynf_metadata_avail()) {
1266                         data->dynf_meta = 0;
1267                         data->flow_meta_mask = 0;
1268                         data->flow_meta_offset = -1;
1269                         data->flow_meta_port_mask = 0;
1270                 } else {
1271                         data->dynf_meta = 1;
1272                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1273                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1274                         data->flow_meta_port_mask = (uint32_t)~0;
1275                         if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16)
1276                                 data->flow_meta_port_mask >>= 16;
1277                 }
1278         }
1279 }
1280
1281 /*
1282  * return a pointer to the desired action in the list of actions.
1283  *
1284  * @param[in] actions
1285  *   The list of actions to search the action in.
1286  * @param[in] action
1287  *   The action to find.
1288  *
1289  * @return
1290  *   Pointer to the action in the list, if found. NULL otherwise.
1291  */
1292 const struct rte_flow_action *
1293 mlx5_flow_find_action(const struct rte_flow_action *actions,
1294                       enum rte_flow_action_type action)
1295 {
1296         if (actions == NULL)
1297                 return NULL;
1298         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1299                 if (actions->type == action)
1300                         return actions;
1301         return NULL;
1302 }
1303
1304 /*
1305  * Validate the flag action.
1306  *
1307  * @param[in] action_flags
1308  *   Bit-fields that holds the actions detected until now.
1309  * @param[in] attr
1310  *   Attributes of flow that includes this action.
1311  * @param[out] error
1312  *   Pointer to error structure.
1313  *
1314  * @return
1315  *   0 on success, a negative errno value otherwise and rte_errno is set.
1316  */
1317 int
1318 mlx5_flow_validate_action_flag(uint64_t action_flags,
1319                                const struct rte_flow_attr *attr,
1320                                struct rte_flow_error *error)
1321 {
1322         if (action_flags & MLX5_FLOW_ACTION_MARK)
1323                 return rte_flow_error_set(error, EINVAL,
1324                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1325                                           "can't mark and flag in same flow");
1326         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1327                 return rte_flow_error_set(error, EINVAL,
1328                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1329                                           "can't have 2 flag"
1330                                           " actions in same flow");
1331         if (attr->egress)
1332                 return rte_flow_error_set(error, ENOTSUP,
1333                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1334                                           "flag action not supported for "
1335                                           "egress");
1336         return 0;
1337 }
1338
1339 /*
1340  * Validate the mark action.
1341  *
1342  * @param[in] action
1343  *   Pointer to the queue action.
1344  * @param[in] action_flags
1345  *   Bit-fields that holds the actions detected until now.
1346  * @param[in] attr
1347  *   Attributes of flow that includes this action.
1348  * @param[out] error
1349  *   Pointer to error structure.
1350  *
1351  * @return
1352  *   0 on success, a negative errno value otherwise and rte_errno is set.
1353  */
1354 int
1355 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1356                                uint64_t action_flags,
1357                                const struct rte_flow_attr *attr,
1358                                struct rte_flow_error *error)
1359 {
1360         const struct rte_flow_action_mark *mark = action->conf;
1361
1362         if (!mark)
1363                 return rte_flow_error_set(error, EINVAL,
1364                                           RTE_FLOW_ERROR_TYPE_ACTION,
1365                                           action,
1366                                           "configuration cannot be null");
1367         if (mark->id >= MLX5_FLOW_MARK_MAX)
1368                 return rte_flow_error_set(error, EINVAL,
1369                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1370                                           &mark->id,
1371                                           "mark id must in 0 <= id < "
1372                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1373         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1374                 return rte_flow_error_set(error, EINVAL,
1375                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1376                                           "can't flag and mark in same flow");
1377         if (action_flags & MLX5_FLOW_ACTION_MARK)
1378                 return rte_flow_error_set(error, EINVAL,
1379                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1380                                           "can't have 2 mark actions in same"
1381                                           " flow");
1382         if (attr->egress)
1383                 return rte_flow_error_set(error, ENOTSUP,
1384                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1385                                           "mark action not supported for "
1386                                           "egress");
1387         return 0;
1388 }
1389
1390 /*
1391  * Validate the drop action.
1392  *
1393  * @param[in] action_flags
1394  *   Bit-fields that holds the actions detected until now.
1395  * @param[in] attr
1396  *   Attributes of flow that includes this action.
1397  * @param[out] error
1398  *   Pointer to error structure.
1399  *
1400  * @return
1401  *   0 on success, a negative errno value otherwise and rte_errno is set.
1402  */
1403 int
1404 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1405                                const struct rte_flow_attr *attr,
1406                                struct rte_flow_error *error)
1407 {
1408         if (attr->egress)
1409                 return rte_flow_error_set(error, ENOTSUP,
1410                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1411                                           "drop action not supported for "
1412                                           "egress");
1413         return 0;
1414 }
1415
1416 /*
1417  * Validate the queue action.
1418  *
1419  * @param[in] action
1420  *   Pointer to the queue action.
1421  * @param[in] action_flags
1422  *   Bit-fields that holds the actions detected until now.
1423  * @param[in] dev
1424  *   Pointer to the Ethernet device structure.
1425  * @param[in] attr
1426  *   Attributes of flow that includes this action.
1427  * @param[out] error
1428  *   Pointer to error structure.
1429  *
1430  * @return
1431  *   0 on success, a negative errno value otherwise and rte_errno is set.
1432  */
1433 int
1434 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1435                                 uint64_t action_flags,
1436                                 struct rte_eth_dev *dev,
1437                                 const struct rte_flow_attr *attr,
1438                                 struct rte_flow_error *error)
1439 {
1440         struct mlx5_priv *priv = dev->data->dev_private;
1441         const struct rte_flow_action_queue *queue = action->conf;
1442
1443         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1444                 return rte_flow_error_set(error, EINVAL,
1445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1446                                           "can't have 2 fate actions in"
1447                                           " same flow");
1448         if (!priv->rxqs_n)
1449                 return rte_flow_error_set(error, EINVAL,
1450                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1451                                           NULL, "No Rx queues configured");
1452         if (queue->index >= priv->rxqs_n)
1453                 return rte_flow_error_set(error, EINVAL,
1454                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1455                                           &queue->index,
1456                                           "queue index out of range");
1457         if (!(*priv->rxqs)[queue->index])
1458                 return rte_flow_error_set(error, EINVAL,
1459                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1460                                           &queue->index,
1461                                           "queue is not configured");
1462         if (attr->egress)
1463                 return rte_flow_error_set(error, ENOTSUP,
1464                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1465                                           "queue action not supported for "
1466                                           "egress");
1467         return 0;
1468 }
1469
1470 /*
1471  * Validate the rss action.
1472  *
1473  * @param[in] dev
1474  *   Pointer to the Ethernet device structure.
1475  * @param[in] action
1476  *   Pointer to the queue action.
1477  * @param[out] error
1478  *   Pointer to error structure.
1479  *
1480  * @return
1481  *   0 on success, a negative errno value otherwise and rte_errno is set.
1482  */
1483 int
1484 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1485                          const struct rte_flow_action *action,
1486                          struct rte_flow_error *error)
1487 {
1488         struct mlx5_priv *priv = dev->data->dev_private;
1489         const struct rte_flow_action_rss *rss = action->conf;
1490         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1491         unsigned int i;
1492
1493         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1494             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1495                 return rte_flow_error_set(error, ENOTSUP,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1497                                           &rss->func,
1498                                           "RSS hash function not supported");
1499 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1500         if (rss->level > 2)
1501 #else
1502         if (rss->level > 1)
1503 #endif
1504                 return rte_flow_error_set(error, ENOTSUP,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1506                                           &rss->level,
1507                                           "tunnel RSS is not supported");
1508         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1509         if (rss->key_len == 0 && rss->key != NULL)
1510                 return rte_flow_error_set(error, ENOTSUP,
1511                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1512                                           &rss->key_len,
1513                                           "RSS hash key length 0");
1514         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1515                 return rte_flow_error_set(error, ENOTSUP,
1516                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1517                                           &rss->key_len,
1518                                           "RSS hash key too small");
1519         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1520                 return rte_flow_error_set(error, ENOTSUP,
1521                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1522                                           &rss->key_len,
1523                                           "RSS hash key too large");
1524         if (rss->queue_num > priv->config.ind_table_max_size)
1525                 return rte_flow_error_set(error, ENOTSUP,
1526                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1527                                           &rss->queue_num,
1528                                           "number of queues too large");
1529         if (rss->types & MLX5_RSS_HF_MASK)
1530                 return rte_flow_error_set(error, ENOTSUP,
1531                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1532                                           &rss->types,
1533                                           "some RSS protocols are not"
1534                                           " supported");
1535         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1536             !(rss->types & ETH_RSS_IP))
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1539                                           "L3 partial RSS requested but L3 RSS"
1540                                           " type not specified");
1541         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1542             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1543                 return rte_flow_error_set(error, EINVAL,
1544                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1545                                           "L4 partial RSS requested but L4 RSS"
1546                                           " type not specified");
1547         if (!priv->rxqs_n)
1548                 return rte_flow_error_set(error, EINVAL,
1549                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1550                                           NULL, "No Rx queues configured");
1551         if (!rss->queue_num)
1552                 return rte_flow_error_set(error, EINVAL,
1553                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1554                                           NULL, "No queues configured");
1555         for (i = 0; i != rss->queue_num; ++i) {
1556                 struct mlx5_rxq_ctrl *rxq_ctrl;
1557
1558                 if (rss->queue[i] >= priv->rxqs_n)
1559                         return rte_flow_error_set
1560                                 (error, EINVAL,
1561                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1562                                  &rss->queue[i], "queue index out of range");
1563                 if (!(*priv->rxqs)[rss->queue[i]])
1564                         return rte_flow_error_set
1565                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1566                                  &rss->queue[i], "queue is not configured");
1567                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1568                                         struct mlx5_rxq_ctrl, rxq);
1569                 if (i == 0)
1570                         rxq_type = rxq_ctrl->type;
1571                 if (rxq_type != rxq_ctrl->type)
1572                         return rte_flow_error_set
1573                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1574                                  &rss->queue[i],
1575                                  "combining hairpin and regular RSS queues is not supported");
1576         }
1577         return 0;
1578 }
1579
1580 /*
1581  * Validate the rss action.
1582  *
1583  * @param[in] action
1584  *   Pointer to the queue action.
1585  * @param[in] action_flags
1586  *   Bit-fields that holds the actions detected until now.
1587  * @param[in] dev
1588  *   Pointer to the Ethernet device structure.
1589  * @param[in] attr
1590  *   Attributes of flow that includes this action.
1591  * @param[in] item_flags
1592  *   Items that were detected.
1593  * @param[out] error
1594  *   Pointer to error structure.
1595  *
1596  * @return
1597  *   0 on success, a negative errno value otherwise and rte_errno is set.
1598  */
1599 int
1600 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1601                               uint64_t action_flags,
1602                               struct rte_eth_dev *dev,
1603                               const struct rte_flow_attr *attr,
1604                               uint64_t item_flags,
1605                               struct rte_flow_error *error)
1606 {
1607         const struct rte_flow_action_rss *rss = action->conf;
1608         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1609         int ret;
1610
1611         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1612                 return rte_flow_error_set(error, EINVAL,
1613                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1614                                           "can't have 2 fate actions"
1615                                           " in same flow");
1616         ret = mlx5_validate_action_rss(dev, action, error);
1617         if (ret)
1618                 return ret;
1619         if (attr->egress)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1622                                           "rss action not supported for "
1623                                           "egress");
1624         if (rss->level > 1 && !tunnel)
1625                 return rte_flow_error_set(error, EINVAL,
1626                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1627                                           "inner RSS is not supported for "
1628                                           "non-tunnel flows");
1629         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1630             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1631                 return rte_flow_error_set(error, EINVAL,
1632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1633                                           "RSS on eCPRI is not supported now");
1634         }
1635         return 0;
1636 }
1637
1638 /*
1639  * Validate the default miss action.
1640  *
1641  * @param[in] action_flags
1642  *   Bit-fields that holds the actions detected until now.
1643  * @param[out] error
1644  *   Pointer to error structure.
1645  *
1646  * @return
1647  *   0 on success, a negative errno value otherwise and rte_errno is set.
1648  */
1649 int
1650 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1651                                 const struct rte_flow_attr *attr,
1652                                 struct rte_flow_error *error)
1653 {
1654         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1655                 return rte_flow_error_set(error, EINVAL,
1656                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1657                                           "can't have 2 fate actions in"
1658                                           " same flow");
1659         if (attr->egress)
1660                 return rte_flow_error_set(error, ENOTSUP,
1661                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1662                                           "default miss action not supported "
1663                                           "for egress");
1664         if (attr->group)
1665                 return rte_flow_error_set(error, ENOTSUP,
1666                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1667                                           "only group 0 is supported");
1668         if (attr->transfer)
1669                 return rte_flow_error_set(error, ENOTSUP,
1670                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1671                                           NULL, "transfer is not supported");
1672         return 0;
1673 }
1674
1675 /*
1676  * Validate the count action.
1677  *
1678  * @param[in] dev
1679  *   Pointer to the Ethernet device structure.
1680  * @param[in] attr
1681  *   Attributes of flow that includes this action.
1682  * @param[out] error
1683  *   Pointer to error structure.
1684  *
1685  * @return
1686  *   0 on success, a negative errno value otherwise and rte_errno is set.
1687  */
1688 int
1689 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1690                                 const struct rte_flow_attr *attr,
1691                                 struct rte_flow_error *error)
1692 {
1693         if (attr->egress)
1694                 return rte_flow_error_set(error, ENOTSUP,
1695                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1696                                           "count action not supported for "
1697                                           "egress");
1698         return 0;
1699 }
1700
1701 /*
1702  * Validate the ASO CT action.
1703  *
1704  * @param[in] dev
1705  *   Pointer to the Ethernet device structure.
1706  * @param[in] conntrack
1707  *   Pointer to the CT action profile.
1708  * @param[out] error
1709  *   Pointer to error structure.
1710  *
1711  * @return
1712  *   0 on success, a negative errno value otherwise and rte_errno is set.
1713  */
1714 int
1715 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1716                         const struct rte_flow_action_conntrack *conntrack,
1717                         struct rte_flow_error *error)
1718 {
1719         RTE_SET_USED(dev);
1720
1721         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1722                 return rte_flow_error_set(error, EINVAL,
1723                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1724                                           "Invalid CT state");
1725         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1726                 return rte_flow_error_set(error, EINVAL,
1727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1728                                           "Invalid last TCP packet flag");
1729         return 0;
1730 }
1731
1732 /**
1733  * Verify the @p attributes will be correctly understood by the NIC and store
1734  * them in the @p flow if everything is correct.
1735  *
1736  * @param[in] dev
1737  *   Pointer to the Ethernet device structure.
1738  * @param[in] attributes
1739  *   Pointer to flow attributes
1740  * @param[out] error
1741  *   Pointer to error structure.
1742  *
1743  * @return
1744  *   0 on success, a negative errno value otherwise and rte_errno is set.
1745  */
1746 int
1747 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1748                               const struct rte_flow_attr *attributes,
1749                               struct rte_flow_error *error)
1750 {
1751         struct mlx5_priv *priv = dev->data->dev_private;
1752         uint32_t priority_max = priv->config.flow_prio - 1;
1753
1754         if (attributes->group)
1755                 return rte_flow_error_set(error, ENOTSUP,
1756                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1757                                           NULL, "groups is not supported");
1758         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1759             attributes->priority >= priority_max)
1760                 return rte_flow_error_set(error, ENOTSUP,
1761                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1762                                           NULL, "priority out of range");
1763         if (attributes->egress)
1764                 return rte_flow_error_set(error, ENOTSUP,
1765                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1766                                           "egress is not supported");
1767         if (attributes->transfer && !priv->config.dv_esw_en)
1768                 return rte_flow_error_set(error, ENOTSUP,
1769                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1770                                           NULL, "transfer is not supported");
1771         if (!attributes->ingress)
1772                 return rte_flow_error_set(error, EINVAL,
1773                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1774                                           NULL,
1775                                           "ingress attribute is mandatory");
1776         return 0;
1777 }
1778
1779 /**
1780  * Validate ICMP6 item.
1781  *
1782  * @param[in] item
1783  *   Item specification.
1784  * @param[in] item_flags
1785  *   Bit-fields that holds the items detected until now.
1786  * @param[in] ext_vlan_sup
1787  *   Whether extended VLAN features are supported or not.
1788  * @param[out] error
1789  *   Pointer to error structure.
1790  *
1791  * @return
1792  *   0 on success, a negative errno value otherwise and rte_errno is set.
1793  */
1794 int
1795 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1796                                uint64_t item_flags,
1797                                uint8_t target_protocol,
1798                                struct rte_flow_error *error)
1799 {
1800         const struct rte_flow_item_icmp6 *mask = item->mask;
1801         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1802         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1803                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1804         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1805                                       MLX5_FLOW_LAYER_OUTER_L4;
1806         int ret;
1807
1808         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1809                 return rte_flow_error_set(error, EINVAL,
1810                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1811                                           "protocol filtering not compatible"
1812                                           " with ICMP6 layer");
1813         if (!(item_flags & l3m))
1814                 return rte_flow_error_set(error, EINVAL,
1815                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1816                                           "IPv6 is mandatory to filter on"
1817                                           " ICMP6");
1818         if (item_flags & l4m)
1819                 return rte_flow_error_set(error, EINVAL,
1820                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1821                                           "multiple L4 layers not supported");
1822         if (!mask)
1823                 mask = &rte_flow_item_icmp6_mask;
1824         ret = mlx5_flow_item_acceptable
1825                 (item, (const uint8_t *)mask,
1826                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1827                  sizeof(struct rte_flow_item_icmp6),
1828                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1829         if (ret < 0)
1830                 return ret;
1831         return 0;
1832 }
1833
1834 /**
1835  * Validate ICMP item.
1836  *
1837  * @param[in] item
1838  *   Item specification.
1839  * @param[in] item_flags
1840  *   Bit-fields that holds the items detected until now.
1841  * @param[out] error
1842  *   Pointer to error structure.
1843  *
1844  * @return
1845  *   0 on success, a negative errno value otherwise and rte_errno is set.
1846  */
1847 int
1848 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1849                              uint64_t item_flags,
1850                              uint8_t target_protocol,
1851                              struct rte_flow_error *error)
1852 {
1853         const struct rte_flow_item_icmp *mask = item->mask;
1854         const struct rte_flow_item_icmp nic_mask = {
1855                 .hdr.icmp_type = 0xff,
1856                 .hdr.icmp_code = 0xff,
1857                 .hdr.icmp_ident = RTE_BE16(0xffff),
1858                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1859         };
1860         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1861         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1862                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1863         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1864                                       MLX5_FLOW_LAYER_OUTER_L4;
1865         int ret;
1866
1867         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1868                 return rte_flow_error_set(error, EINVAL,
1869                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1870                                           "protocol filtering not compatible"
1871                                           " with ICMP layer");
1872         if (!(item_flags & l3m))
1873                 return rte_flow_error_set(error, EINVAL,
1874                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1875                                           "IPv4 is mandatory to filter"
1876                                           " on ICMP");
1877         if (item_flags & l4m)
1878                 return rte_flow_error_set(error, EINVAL,
1879                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1880                                           "multiple L4 layers not supported");
1881         if (!mask)
1882                 mask = &nic_mask;
1883         ret = mlx5_flow_item_acceptable
1884                 (item, (const uint8_t *)mask,
1885                  (const uint8_t *)&nic_mask,
1886                  sizeof(struct rte_flow_item_icmp),
1887                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1888         if (ret < 0)
1889                 return ret;
1890         return 0;
1891 }
1892
1893 /**
1894  * Validate Ethernet item.
1895  *
1896  * @param[in] item
1897  *   Item specification.
1898  * @param[in] item_flags
1899  *   Bit-fields that holds the items detected until now.
1900  * @param[out] error
1901  *   Pointer to error structure.
1902  *
1903  * @return
1904  *   0 on success, a negative errno value otherwise and rte_errno is set.
1905  */
1906 int
1907 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1908                             uint64_t item_flags, bool ext_vlan_sup,
1909                             struct rte_flow_error *error)
1910 {
1911         const struct rte_flow_item_eth *mask = item->mask;
1912         const struct rte_flow_item_eth nic_mask = {
1913                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1914                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1915                 .type = RTE_BE16(0xffff),
1916                 .has_vlan = ext_vlan_sup ? 1 : 0,
1917         };
1918         int ret;
1919         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1920         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1921                                        MLX5_FLOW_LAYER_OUTER_L2;
1922
1923         if (item_flags & ethm)
1924                 return rte_flow_error_set(error, ENOTSUP,
1925                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1926                                           "multiple L2 layers not supported");
1927         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1928             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1929                 return rte_flow_error_set(error, EINVAL,
1930                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1931                                           "L2 layer should not follow "
1932                                           "L3 layers");
1933         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1934             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1935                 return rte_flow_error_set(error, EINVAL,
1936                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1937                                           "L2 layer should not follow VLAN");
1938         if (!mask)
1939                 mask = &rte_flow_item_eth_mask;
1940         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1941                                         (const uint8_t *)&nic_mask,
1942                                         sizeof(struct rte_flow_item_eth),
1943                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1944         return ret;
1945 }
1946
1947 /**
1948  * Validate VLAN item.
1949  *
1950  * @param[in] item
1951  *   Item specification.
1952  * @param[in] item_flags
1953  *   Bit-fields that holds the items detected until now.
1954  * @param[in] dev
1955  *   Ethernet device flow is being created on.
1956  * @param[out] error
1957  *   Pointer to error structure.
1958  *
1959  * @return
1960  *   0 on success, a negative errno value otherwise and rte_errno is set.
1961  */
1962 int
1963 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1964                              uint64_t item_flags,
1965                              struct rte_eth_dev *dev,
1966                              struct rte_flow_error *error)
1967 {
1968         const struct rte_flow_item_vlan *spec = item->spec;
1969         const struct rte_flow_item_vlan *mask = item->mask;
1970         const struct rte_flow_item_vlan nic_mask = {
1971                 .tci = RTE_BE16(UINT16_MAX),
1972                 .inner_type = RTE_BE16(UINT16_MAX),
1973         };
1974         uint16_t vlan_tag = 0;
1975         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1976         int ret;
1977         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1978                                         MLX5_FLOW_LAYER_INNER_L4) :
1979                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1980                                         MLX5_FLOW_LAYER_OUTER_L4);
1981         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1982                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1983
1984         if (item_flags & vlanm)
1985                 return rte_flow_error_set(error, EINVAL,
1986                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1987                                           "multiple VLAN layers not supported");
1988         else if ((item_flags & l34m) != 0)
1989                 return rte_flow_error_set(error, EINVAL,
1990                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1991                                           "VLAN cannot follow L3/L4 layer");
1992         if (!mask)
1993                 mask = &rte_flow_item_vlan_mask;
1994         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1995                                         (const uint8_t *)&nic_mask,
1996                                         sizeof(struct rte_flow_item_vlan),
1997                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1998         if (ret)
1999                 return ret;
2000         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2001                 struct mlx5_priv *priv = dev->data->dev_private;
2002
2003                 if (priv->vmwa_context) {
2004                         /*
2005                          * Non-NULL context means we have a virtual machine
2006                          * and SR-IOV enabled, we have to create VLAN interface
2007                          * to make hypervisor to setup E-Switch vport
2008                          * context correctly. We avoid creating the multiple
2009                          * VLAN interfaces, so we cannot support VLAN tag mask.
2010                          */
2011                         return rte_flow_error_set(error, EINVAL,
2012                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2013                                                   item,
2014                                                   "VLAN tag mask is not"
2015                                                   " supported in virtual"
2016                                                   " environment");
2017                 }
2018         }
2019         if (spec) {
2020                 vlan_tag = spec->tci;
2021                 vlan_tag &= mask->tci;
2022         }
2023         /*
2024          * From verbs perspective an empty VLAN is equivalent
2025          * to a packet without VLAN layer.
2026          */
2027         if (!vlan_tag)
2028                 return rte_flow_error_set(error, EINVAL,
2029                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2030                                           item->spec,
2031                                           "VLAN cannot be empty");
2032         return 0;
2033 }
2034
2035 /**
2036  * Validate IPV4 item.
2037  *
2038  * @param[in] item
2039  *   Item specification.
2040  * @param[in] item_flags
2041  *   Bit-fields that holds the items detected until now.
2042  * @param[in] last_item
2043  *   Previous validated item in the pattern items.
2044  * @param[in] ether_type
2045  *   Type in the ethernet layer header (including dot1q).
2046  * @param[in] acc_mask
2047  *   Acceptable mask, if NULL default internal default mask
2048  *   will be used to check whether item fields are supported.
2049  * @param[in] range_accepted
2050  *   True if range of values is accepted for specific fields, false otherwise.
2051  * @param[out] error
2052  *   Pointer to error structure.
2053  *
2054  * @return
2055  *   0 on success, a negative errno value otherwise and rte_errno is set.
2056  */
2057 int
2058 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2059                              uint64_t item_flags,
2060                              uint64_t last_item,
2061                              uint16_t ether_type,
2062                              const struct rte_flow_item_ipv4 *acc_mask,
2063                              bool range_accepted,
2064                              struct rte_flow_error *error)
2065 {
2066         const struct rte_flow_item_ipv4 *mask = item->mask;
2067         const struct rte_flow_item_ipv4 *spec = item->spec;
2068         const struct rte_flow_item_ipv4 nic_mask = {
2069                 .hdr = {
2070                         .src_addr = RTE_BE32(0xffffffff),
2071                         .dst_addr = RTE_BE32(0xffffffff),
2072                         .type_of_service = 0xff,
2073                         .next_proto_id = 0xff,
2074                 },
2075         };
2076         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2077         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2078                                       MLX5_FLOW_LAYER_OUTER_L3;
2079         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2080                                       MLX5_FLOW_LAYER_OUTER_L4;
2081         int ret;
2082         uint8_t next_proto = 0xFF;
2083         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2084                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2085                                   MLX5_FLOW_LAYER_INNER_VLAN);
2086
2087         if ((last_item & l2_vlan) && ether_type &&
2088             ether_type != RTE_ETHER_TYPE_IPV4)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2091                                           "IPv4 cannot follow L2/VLAN layer "
2092                                           "which ether type is not IPv4");
2093         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2094                 if (mask && spec)
2095                         next_proto = mask->hdr.next_proto_id &
2096                                      spec->hdr.next_proto_id;
2097                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2098                         return rte_flow_error_set(error, EINVAL,
2099                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2100                                                   item,
2101                                                   "multiple tunnel "
2102                                                   "not supported");
2103         }
2104         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2105                 return rte_flow_error_set(error, EINVAL,
2106                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2107                                           "wrong tunnel type - IPv6 specified "
2108                                           "but IPv4 item provided");
2109         if (item_flags & l3m)
2110                 return rte_flow_error_set(error, ENOTSUP,
2111                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2112                                           "multiple L3 layers not supported");
2113         else if (item_flags & l4m)
2114                 return rte_flow_error_set(error, EINVAL,
2115                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2116                                           "L3 cannot follow an L4 layer.");
2117         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2118                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2119                 return rte_flow_error_set(error, EINVAL,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2121                                           "L3 cannot follow an NVGRE layer.");
2122         if (!mask)
2123                 mask = &rte_flow_item_ipv4_mask;
2124         else if (mask->hdr.next_proto_id != 0 &&
2125                  mask->hdr.next_proto_id != 0xff)
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2128                                           "partial mask is not supported"
2129                                           " for protocol");
2130         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2131                                         acc_mask ? (const uint8_t *)acc_mask
2132                                                  : (const uint8_t *)&nic_mask,
2133                                         sizeof(struct rte_flow_item_ipv4),
2134                                         range_accepted, error);
2135         if (ret < 0)
2136                 return ret;
2137         return 0;
2138 }
2139
2140 /**
2141  * Validate IPV6 item.
2142  *
2143  * @param[in] item
2144  *   Item specification.
2145  * @param[in] item_flags
2146  *   Bit-fields that holds the items detected until now.
2147  * @param[in] last_item
2148  *   Previous validated item in the pattern items.
2149  * @param[in] ether_type
2150  *   Type in the ethernet layer header (including dot1q).
2151  * @param[in] acc_mask
2152  *   Acceptable mask, if NULL default internal default mask
2153  *   will be used to check whether item fields are supported.
2154  * @param[out] error
2155  *   Pointer to error structure.
2156  *
2157  * @return
2158  *   0 on success, a negative errno value otherwise and rte_errno is set.
2159  */
2160 int
2161 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2162                              uint64_t item_flags,
2163                              uint64_t last_item,
2164                              uint16_t ether_type,
2165                              const struct rte_flow_item_ipv6 *acc_mask,
2166                              struct rte_flow_error *error)
2167 {
2168         const struct rte_flow_item_ipv6 *mask = item->mask;
2169         const struct rte_flow_item_ipv6 *spec = item->spec;
2170         const struct rte_flow_item_ipv6 nic_mask = {
2171                 .hdr = {
2172                         .src_addr =
2173                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2174                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2175                         .dst_addr =
2176                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2177                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2178                         .vtc_flow = RTE_BE32(0xffffffff),
2179                         .proto = 0xff,
2180                 },
2181         };
2182         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2183         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2184                                       MLX5_FLOW_LAYER_OUTER_L3;
2185         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2186                                       MLX5_FLOW_LAYER_OUTER_L4;
2187         int ret;
2188         uint8_t next_proto = 0xFF;
2189         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2190                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2191                                   MLX5_FLOW_LAYER_INNER_VLAN);
2192
2193         if ((last_item & l2_vlan) && ether_type &&
2194             ether_type != RTE_ETHER_TYPE_IPV6)
2195                 return rte_flow_error_set(error, EINVAL,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2197                                           "IPv6 cannot follow L2/VLAN layer "
2198                                           "which ether type is not IPv6");
2199         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2200                 next_proto = spec->hdr.proto;
2201         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2202                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2203                         return rte_flow_error_set(error, EINVAL,
2204                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2205                                                   item,
2206                                                   "multiple tunnel "
2207                                                   "not supported");
2208         }
2209         if (next_proto == IPPROTO_HOPOPTS  ||
2210             next_proto == IPPROTO_ROUTING  ||
2211             next_proto == IPPROTO_FRAGMENT ||
2212             next_proto == IPPROTO_ESP      ||
2213             next_proto == IPPROTO_AH       ||
2214             next_proto == IPPROTO_DSTOPTS)
2215                 return rte_flow_error_set(error, EINVAL,
2216                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2217                                           "IPv6 proto (next header) should "
2218                                           "not be set as extension header");
2219         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2220                 return rte_flow_error_set(error, EINVAL,
2221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2222                                           "wrong tunnel type - IPv4 specified "
2223                                           "but IPv6 item provided");
2224         if (item_flags & l3m)
2225                 return rte_flow_error_set(error, ENOTSUP,
2226                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2227                                           "multiple L3 layers not supported");
2228         else if (item_flags & l4m)
2229                 return rte_flow_error_set(error, EINVAL,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2231                                           "L3 cannot follow an L4 layer.");
2232         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2233                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2234                 return rte_flow_error_set(error, EINVAL,
2235                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2236                                           "L3 cannot follow an NVGRE layer.");
2237         if (!mask)
2238                 mask = &rte_flow_item_ipv6_mask;
2239         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2240                                         acc_mask ? (const uint8_t *)acc_mask
2241                                                  : (const uint8_t *)&nic_mask,
2242                                         sizeof(struct rte_flow_item_ipv6),
2243                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2244         if (ret < 0)
2245                 return ret;
2246         return 0;
2247 }
2248
2249 /**
2250  * Validate UDP item.
2251  *
2252  * @param[in] item
2253  *   Item specification.
2254  * @param[in] item_flags
2255  *   Bit-fields that holds the items detected until now.
2256  * @param[in] target_protocol
2257  *   The next protocol in the previous item.
2258  * @param[in] flow_mask
2259  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2260  * @param[out] error
2261  *   Pointer to error structure.
2262  *
2263  * @return
2264  *   0 on success, a negative errno value otherwise and rte_errno is set.
2265  */
2266 int
2267 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2268                             uint64_t item_flags,
2269                             uint8_t target_protocol,
2270                             struct rte_flow_error *error)
2271 {
2272         const struct rte_flow_item_udp *mask = item->mask;
2273         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2274         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2275                                       MLX5_FLOW_LAYER_OUTER_L3;
2276         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2277                                       MLX5_FLOW_LAYER_OUTER_L4;
2278         int ret;
2279
2280         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2281                 return rte_flow_error_set(error, EINVAL,
2282                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2283                                           "protocol filtering not compatible"
2284                                           " with UDP layer");
2285         if (!(item_flags & l3m))
2286                 return rte_flow_error_set(error, EINVAL,
2287                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2288                                           "L3 is mandatory to filter on L4");
2289         if (item_flags & l4m)
2290                 return rte_flow_error_set(error, EINVAL,
2291                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2292                                           "multiple L4 layers not supported");
2293         if (!mask)
2294                 mask = &rte_flow_item_udp_mask;
2295         ret = mlx5_flow_item_acceptable
2296                 (item, (const uint8_t *)mask,
2297                  (const uint8_t *)&rte_flow_item_udp_mask,
2298                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2299                  error);
2300         if (ret < 0)
2301                 return ret;
2302         return 0;
2303 }
2304
2305 /**
2306  * Validate TCP item.
2307  *
2308  * @param[in] item
2309  *   Item specification.
2310  * @param[in] item_flags
2311  *   Bit-fields that holds the items detected until now.
2312  * @param[in] target_protocol
2313  *   The next protocol in the previous item.
2314  * @param[out] error
2315  *   Pointer to error structure.
2316  *
2317  * @return
2318  *   0 on success, a negative errno value otherwise and rte_errno is set.
2319  */
2320 int
2321 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2322                             uint64_t item_flags,
2323                             uint8_t target_protocol,
2324                             const struct rte_flow_item_tcp *flow_mask,
2325                             struct rte_flow_error *error)
2326 {
2327         const struct rte_flow_item_tcp *mask = item->mask;
2328         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2329         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2330                                       MLX5_FLOW_LAYER_OUTER_L3;
2331         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2332                                       MLX5_FLOW_LAYER_OUTER_L4;
2333         int ret;
2334
2335         MLX5_ASSERT(flow_mask);
2336         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2337                 return rte_flow_error_set(error, EINVAL,
2338                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2339                                           "protocol filtering not compatible"
2340                                           " with TCP layer");
2341         if (!(item_flags & l3m))
2342                 return rte_flow_error_set(error, EINVAL,
2343                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2344                                           "L3 is mandatory to filter on L4");
2345         if (item_flags & l4m)
2346                 return rte_flow_error_set(error, EINVAL,
2347                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2348                                           "multiple L4 layers not supported");
2349         if (!mask)
2350                 mask = &rte_flow_item_tcp_mask;
2351         ret = mlx5_flow_item_acceptable
2352                 (item, (const uint8_t *)mask,
2353                  (const uint8_t *)flow_mask,
2354                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2355                  error);
2356         if (ret < 0)
2357                 return ret;
2358         return 0;
2359 }
2360
2361 /**
2362  * Validate VXLAN item.
2363  *
2364  * @param[in] item
2365  *   Item specification.
2366  * @param[in] item_flags
2367  *   Bit-fields that holds the items detected until now.
2368  * @param[in] target_protocol
2369  *   The next protocol in the previous item.
2370  * @param[out] error
2371  *   Pointer to error structure.
2372  *
2373  * @return
2374  *   0 on success, a negative errno value otherwise and rte_errno is set.
2375  */
2376 int
2377 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2378                               uint64_t item_flags,
2379                               struct rte_flow_error *error)
2380 {
2381         const struct rte_flow_item_vxlan *spec = item->spec;
2382         const struct rte_flow_item_vxlan *mask = item->mask;
2383         int ret;
2384         union vni {
2385                 uint32_t vlan_id;
2386                 uint8_t vni[4];
2387         } id = { .vlan_id = 0, };
2388
2389
2390         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2391                 return rte_flow_error_set(error, ENOTSUP,
2392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2393                                           "multiple tunnel layers not"
2394                                           " supported");
2395         /*
2396          * Verify only UDPv4 is present as defined in
2397          * https://tools.ietf.org/html/rfc7348
2398          */
2399         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2400                 return rte_flow_error_set(error, EINVAL,
2401                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2402                                           "no outer UDP layer found");
2403         if (!mask)
2404                 mask = &rte_flow_item_vxlan_mask;
2405         ret = mlx5_flow_item_acceptable
2406                 (item, (const uint8_t *)mask,
2407                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2408                  sizeof(struct rte_flow_item_vxlan),
2409                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2410         if (ret < 0)
2411                 return ret;
2412         if (spec) {
2413                 memcpy(&id.vni[1], spec->vni, 3);
2414                 memcpy(&id.vni[1], mask->vni, 3);
2415         }
2416         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2419                                           "VXLAN tunnel must be fully defined");
2420         return 0;
2421 }
2422
2423 /**
2424  * Validate VXLAN_GPE item.
2425  *
2426  * @param[in] item
2427  *   Item specification.
2428  * @param[in] item_flags
2429  *   Bit-fields that holds the items detected until now.
2430  * @param[in] priv
2431  *   Pointer to the private data structure.
2432  * @param[in] target_protocol
2433  *   The next protocol in the previous item.
2434  * @param[out] error
2435  *   Pointer to error structure.
2436  *
2437  * @return
2438  *   0 on success, a negative errno value otherwise and rte_errno is set.
2439  */
2440 int
2441 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2442                                   uint64_t item_flags,
2443                                   struct rte_eth_dev *dev,
2444                                   struct rte_flow_error *error)
2445 {
2446         struct mlx5_priv *priv = dev->data->dev_private;
2447         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2448         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2449         int ret;
2450         union vni {
2451                 uint32_t vlan_id;
2452                 uint8_t vni[4];
2453         } id = { .vlan_id = 0, };
2454
2455         if (!priv->config.l3_vxlan_en)
2456                 return rte_flow_error_set(error, ENOTSUP,
2457                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2458                                           "L3 VXLAN is not enabled by device"
2459                                           " parameter and/or not configured in"
2460                                           " firmware");
2461         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2462                 return rte_flow_error_set(error, ENOTSUP,
2463                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2464                                           "multiple tunnel layers not"
2465                                           " supported");
2466         /*
2467          * Verify only UDPv4 is present as defined in
2468          * https://tools.ietf.org/html/rfc7348
2469          */
2470         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2471                 return rte_flow_error_set(error, EINVAL,
2472                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2473                                           "no outer UDP layer found");
2474         if (!mask)
2475                 mask = &rte_flow_item_vxlan_gpe_mask;
2476         ret = mlx5_flow_item_acceptable
2477                 (item, (const uint8_t *)mask,
2478                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2479                  sizeof(struct rte_flow_item_vxlan_gpe),
2480                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2481         if (ret < 0)
2482                 return ret;
2483         if (spec) {
2484                 if (spec->protocol)
2485                         return rte_flow_error_set(error, ENOTSUP,
2486                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2487                                                   item,
2488                                                   "VxLAN-GPE protocol"
2489                                                   " not supported");
2490                 memcpy(&id.vni[1], spec->vni, 3);
2491                 memcpy(&id.vni[1], mask->vni, 3);
2492         }
2493         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2494                 return rte_flow_error_set(error, ENOTSUP,
2495                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2496                                           "VXLAN-GPE tunnel must be fully"
2497                                           " defined");
2498         return 0;
2499 }
2500 /**
2501  * Validate GRE Key item.
2502  *
2503  * @param[in] item
2504  *   Item specification.
2505  * @param[in] item_flags
2506  *   Bit flags to mark detected items.
2507  * @param[in] gre_item
2508  *   Pointer to gre_item
2509  * @param[out] error
2510  *   Pointer to error structure.
2511  *
2512  * @return
2513  *   0 on success, a negative errno value otherwise and rte_errno is set.
2514  */
2515 int
2516 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2517                                 uint64_t item_flags,
2518                                 const struct rte_flow_item *gre_item,
2519                                 struct rte_flow_error *error)
2520 {
2521         const rte_be32_t *mask = item->mask;
2522         int ret = 0;
2523         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2524         const struct rte_flow_item_gre *gre_spec;
2525         const struct rte_flow_item_gre *gre_mask;
2526
2527         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2528                 return rte_flow_error_set(error, ENOTSUP,
2529                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2530                                           "Multiple GRE key not support");
2531         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2532                 return rte_flow_error_set(error, ENOTSUP,
2533                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2534                                           "No preceding GRE header");
2535         if (item_flags & MLX5_FLOW_LAYER_INNER)
2536                 return rte_flow_error_set(error, ENOTSUP,
2537                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2538                                           "GRE key following a wrong item");
2539         gre_mask = gre_item->mask;
2540         if (!gre_mask)
2541                 gre_mask = &rte_flow_item_gre_mask;
2542         gre_spec = gre_item->spec;
2543         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2544                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2545                 return rte_flow_error_set(error, EINVAL,
2546                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2547                                           "Key bit must be on");
2548
2549         if (!mask)
2550                 mask = &gre_key_default_mask;
2551         ret = mlx5_flow_item_acceptable
2552                 (item, (const uint8_t *)mask,
2553                  (const uint8_t *)&gre_key_default_mask,
2554                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2555         return ret;
2556 }
2557
2558 /**
2559  * Validate GRE item.
2560  *
2561  * @param[in] item
2562  *   Item specification.
2563  * @param[in] item_flags
2564  *   Bit flags to mark detected items.
2565  * @param[in] target_protocol
2566  *   The next protocol in the previous item.
2567  * @param[out] error
2568  *   Pointer to error structure.
2569  *
2570  * @return
2571  *   0 on success, a negative errno value otherwise and rte_errno is set.
2572  */
2573 int
2574 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2575                             uint64_t item_flags,
2576                             uint8_t target_protocol,
2577                             struct rte_flow_error *error)
2578 {
2579         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2580         const struct rte_flow_item_gre *mask = item->mask;
2581         int ret;
2582         const struct rte_flow_item_gre nic_mask = {
2583                 .c_rsvd0_ver = RTE_BE16(0xB000),
2584                 .protocol = RTE_BE16(UINT16_MAX),
2585         };
2586
2587         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2588                 return rte_flow_error_set(error, EINVAL,
2589                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2590                                           "protocol filtering not compatible"
2591                                           " with this GRE layer");
2592         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2593                 return rte_flow_error_set(error, ENOTSUP,
2594                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2595                                           "multiple tunnel layers not"
2596                                           " supported");
2597         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2598                 return rte_flow_error_set(error, ENOTSUP,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2600                                           "L3 Layer is missing");
2601         if (!mask)
2602                 mask = &rte_flow_item_gre_mask;
2603         ret = mlx5_flow_item_acceptable
2604                 (item, (const uint8_t *)mask,
2605                  (const uint8_t *)&nic_mask,
2606                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2607                  error);
2608         if (ret < 0)
2609                 return ret;
2610 #ifndef HAVE_MLX5DV_DR
2611 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2612         if (spec && (spec->protocol & mask->protocol))
2613                 return rte_flow_error_set(error, ENOTSUP,
2614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2615                                           "without MPLS support the"
2616                                           " specification cannot be used for"
2617                                           " filtering");
2618 #endif
2619 #endif
2620         return 0;
2621 }
2622
2623 /**
2624  * Validate Geneve item.
2625  *
2626  * @param[in] item
2627  *   Item specification.
2628  * @param[in] itemFlags
2629  *   Bit-fields that holds the items detected until now.
2630  * @param[in] enPriv
2631  *   Pointer to the private data structure.
2632  * @param[out] error
2633  *   Pointer to error structure.
2634  *
2635  * @return
2636  *   0 on success, a negative errno value otherwise and rte_errno is set.
2637  */
2638
2639 int
2640 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2641                                uint64_t item_flags,
2642                                struct rte_eth_dev *dev,
2643                                struct rte_flow_error *error)
2644 {
2645         struct mlx5_priv *priv = dev->data->dev_private;
2646         const struct rte_flow_item_geneve *spec = item->spec;
2647         const struct rte_flow_item_geneve *mask = item->mask;
2648         int ret;
2649         uint16_t gbhdr;
2650         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2651                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2652         const struct rte_flow_item_geneve nic_mask = {
2653                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2654                 .vni = "\xff\xff\xff",
2655                 .protocol = RTE_BE16(UINT16_MAX),
2656         };
2657
2658         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2659                 return rte_flow_error_set(error, ENOTSUP,
2660                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2661                                           "L3 Geneve is not enabled by device"
2662                                           " parameter and/or not configured in"
2663                                           " firmware");
2664         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2665                 return rte_flow_error_set(error, ENOTSUP,
2666                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2667                                           "multiple tunnel layers not"
2668                                           " supported");
2669         /*
2670          * Verify only UDPv4 is present as defined in
2671          * https://tools.ietf.org/html/rfc7348
2672          */
2673         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2674                 return rte_flow_error_set(error, EINVAL,
2675                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2676                                           "no outer UDP layer found");
2677         if (!mask)
2678                 mask = &rte_flow_item_geneve_mask;
2679         ret = mlx5_flow_item_acceptable
2680                                   (item, (const uint8_t *)mask,
2681                                    (const uint8_t *)&nic_mask,
2682                                    sizeof(struct rte_flow_item_geneve),
2683                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2684         if (ret)
2685                 return ret;
2686         if (spec) {
2687                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2688                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2689                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2690                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2691                         return rte_flow_error_set(error, ENOTSUP,
2692                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2693                                                   item,
2694                                                   "Geneve protocol unsupported"
2695                                                   " fields are being used");
2696                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2697                         return rte_flow_error_set
2698                                         (error, ENOTSUP,
2699                                          RTE_FLOW_ERROR_TYPE_ITEM,
2700                                          item,
2701                                          "Unsupported Geneve options length");
2702         }
2703         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2704                 return rte_flow_error_set
2705                                     (error, ENOTSUP,
2706                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2707                                      "Geneve tunnel must be fully defined");
2708         return 0;
2709 }
2710
2711 /**
2712  * Validate Geneve TLV option item.
2713  *
2714  * @param[in] item
2715  *   Item specification.
2716  * @param[in] last_item
2717  *   Previous validated item in the pattern items.
2718  * @param[in] geneve_item
2719  *   Previous GENEVE item specification.
2720  * @param[in] dev
2721  *   Pointer to the rte_eth_dev structure.
2722  * @param[out] error
2723  *   Pointer to error structure.
2724  *
2725  * @return
2726  *   0 on success, a negative errno value otherwise and rte_errno is set.
2727  */
2728 int
2729 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2730                                    uint64_t last_item,
2731                                    const struct rte_flow_item *geneve_item,
2732                                    struct rte_eth_dev *dev,
2733                                    struct rte_flow_error *error)
2734 {
2735         struct mlx5_priv *priv = dev->data->dev_private;
2736         struct mlx5_dev_ctx_shared *sh = priv->sh;
2737         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2738         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2739         uint8_t data_max_supported =
2740                         hca_attr->max_geneve_tlv_option_data_len * 4;
2741         struct mlx5_dev_config *config = &priv->config;
2742         const struct rte_flow_item_geneve *geneve_spec;
2743         const struct rte_flow_item_geneve *geneve_mask;
2744         const struct rte_flow_item_geneve_opt *spec = item->spec;
2745         const struct rte_flow_item_geneve_opt *mask = item->mask;
2746         unsigned int i;
2747         unsigned int data_len;
2748         uint8_t tlv_option_len;
2749         uint16_t optlen_m, optlen_v;
2750         const struct rte_flow_item_geneve_opt full_mask = {
2751                 .option_class = RTE_BE16(0xffff),
2752                 .option_type = 0xff,
2753                 .option_len = 0x1f,
2754         };
2755
2756         if (!mask)
2757                 mask = &rte_flow_item_geneve_opt_mask;
2758         if (!spec)
2759                 return rte_flow_error_set
2760                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2761                         "Geneve TLV opt class/type/length must be specified");
2762         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2763                 return rte_flow_error_set
2764                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2765                         "Geneve TLV opt length exceeeds the limit (31)");
2766         /* Check if class type and length masks are full. */
2767         if (full_mask.option_class != mask->option_class ||
2768             full_mask.option_type != mask->option_type ||
2769             full_mask.option_len != (mask->option_len & full_mask.option_len))
2770                 return rte_flow_error_set
2771                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                         "Geneve TLV opt class/type/length masks must be full");
2773         /* Check if length is supported */
2774         if ((uint32_t)spec->option_len >
2775                         config->hca_attr.max_geneve_tlv_option_data_len)
2776                 return rte_flow_error_set
2777                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2778                         "Geneve TLV opt length not supported");
2779         if (config->hca_attr.max_geneve_tlv_options > 1)
2780                 DRV_LOG(DEBUG,
2781                         "max_geneve_tlv_options supports more than 1 option");
2782         /* Check GENEVE item preceding. */
2783         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2784                 return rte_flow_error_set
2785                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2786                         "Geneve opt item must be preceded with Geneve item");
2787         geneve_spec = geneve_item->spec;
2788         geneve_mask = geneve_item->mask ? geneve_item->mask :
2789                                           &rte_flow_item_geneve_mask;
2790         /* Check if GENEVE TLV option size doesn't exceed option length */
2791         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2792                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2793                 tlv_option_len = spec->option_len & mask->option_len;
2794                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2795                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2796                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2797                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2798                 if ((optlen_v & optlen_m) <= tlv_option_len)
2799                         return rte_flow_error_set
2800                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2801                                  "GENEVE TLV option length exceeds optlen");
2802         }
2803         /* Check if length is 0 or data is 0. */
2804         if (spec->data == NULL || spec->option_len == 0)
2805                 return rte_flow_error_set
2806                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2807                         "Geneve TLV opt with zero data/length not supported");
2808         /* Check not all data & mask are 0. */
2809         data_len = spec->option_len * 4;
2810         if (mask->data == NULL) {
2811                 for (i = 0; i < data_len; i++)
2812                         if (spec->data[i])
2813                                 break;
2814                 if (i == data_len)
2815                         return rte_flow_error_set(error, ENOTSUP,
2816                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2817                                 "Can't match on Geneve option data 0");
2818         } else {
2819                 for (i = 0; i < data_len; i++)
2820                         if (spec->data[i] & mask->data[i])
2821                                 break;
2822                 if (i == data_len)
2823                         return rte_flow_error_set(error, ENOTSUP,
2824                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2825                                 "Can't match on Geneve option data and mask 0");
2826                 /* Check data mask supported. */
2827                 for (i = data_max_supported; i < data_len ; i++)
2828                         if (mask->data[i])
2829                                 return rte_flow_error_set(error, ENOTSUP,
2830                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2831                                         "Data mask is of unsupported size");
2832         }
2833         /* Check GENEVE option is supported in NIC. */
2834         if (!config->hca_attr.geneve_tlv_opt)
2835                 return rte_flow_error_set
2836                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2837                         "Geneve TLV opt not supported");
2838         /* Check if we already have geneve option with different type/class. */
2839         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2840         geneve_opt_resource = sh->geneve_tlv_option_resource;
2841         if (geneve_opt_resource != NULL)
2842                 if (geneve_opt_resource->option_class != spec->option_class ||
2843                     geneve_opt_resource->option_type != spec->option_type ||
2844                     geneve_opt_resource->length != spec->option_len) {
2845                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2846                         return rte_flow_error_set(error, ENOTSUP,
2847                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2848                                 "Only one Geneve TLV option supported");
2849                 }
2850         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2851         return 0;
2852 }
2853
2854 /**
2855  * Validate MPLS item.
2856  *
2857  * @param[in] dev
2858  *   Pointer to the rte_eth_dev structure.
2859  * @param[in] item
2860  *   Item specification.
2861  * @param[in] item_flags
2862  *   Bit-fields that holds the items detected until now.
2863  * @param[in] prev_layer
2864  *   The protocol layer indicated in previous item.
2865  * @param[out] error
2866  *   Pointer to error structure.
2867  *
2868  * @return
2869  *   0 on success, a negative errno value otherwise and rte_errno is set.
2870  */
2871 int
2872 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2873                              const struct rte_flow_item *item __rte_unused,
2874                              uint64_t item_flags __rte_unused,
2875                              uint64_t prev_layer __rte_unused,
2876                              struct rte_flow_error *error)
2877 {
2878 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2879         const struct rte_flow_item_mpls *mask = item->mask;
2880         struct mlx5_priv *priv = dev->data->dev_private;
2881         int ret;
2882
2883         if (!priv->config.mpls_en)
2884                 return rte_flow_error_set(error, ENOTSUP,
2885                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2886                                           "MPLS not supported or"
2887                                           " disabled in firmware"
2888                                           " configuration.");
2889         /* MPLS over IP, UDP, GRE is allowed */
2890         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2891                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2892                             MLX5_FLOW_LAYER_GRE |
2893                             MLX5_FLOW_LAYER_GRE_KEY)))
2894                 return rte_flow_error_set(error, EINVAL,
2895                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2896                                           "protocol filtering not compatible"
2897                                           " with MPLS layer");
2898         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2899         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2900             !(item_flags & MLX5_FLOW_LAYER_GRE))
2901                 return rte_flow_error_set(error, ENOTSUP,
2902                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2903                                           "multiple tunnel layers not"
2904                                           " supported");
2905         if (!mask)
2906                 mask = &rte_flow_item_mpls_mask;
2907         ret = mlx5_flow_item_acceptable
2908                 (item, (const uint8_t *)mask,
2909                  (const uint8_t *)&rte_flow_item_mpls_mask,
2910                  sizeof(struct rte_flow_item_mpls),
2911                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2912         if (ret < 0)
2913                 return ret;
2914         return 0;
2915 #else
2916         return rte_flow_error_set(error, ENOTSUP,
2917                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2918                                   "MPLS is not supported by Verbs, please"
2919                                   " update.");
2920 #endif
2921 }
2922
2923 /**
2924  * Validate NVGRE item.
2925  *
2926  * @param[in] item
2927  *   Item specification.
2928  * @param[in] item_flags
2929  *   Bit flags to mark detected items.
2930  * @param[in] target_protocol
2931  *   The next protocol in the previous item.
2932  * @param[out] error
2933  *   Pointer to error structure.
2934  *
2935  * @return
2936  *   0 on success, a negative errno value otherwise and rte_errno is set.
2937  */
2938 int
2939 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2940                               uint64_t item_flags,
2941                               uint8_t target_protocol,
2942                               struct rte_flow_error *error)
2943 {
2944         const struct rte_flow_item_nvgre *mask = item->mask;
2945         int ret;
2946
2947         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2948                 return rte_flow_error_set(error, EINVAL,
2949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2950                                           "protocol filtering not compatible"
2951                                           " with this GRE layer");
2952         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2953                 return rte_flow_error_set(error, ENOTSUP,
2954                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2955                                           "multiple tunnel layers not"
2956                                           " supported");
2957         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2958                 return rte_flow_error_set(error, ENOTSUP,
2959                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2960                                           "L3 Layer is missing");
2961         if (!mask)
2962                 mask = &rte_flow_item_nvgre_mask;
2963         ret = mlx5_flow_item_acceptable
2964                 (item, (const uint8_t *)mask,
2965                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2966                  sizeof(struct rte_flow_item_nvgre),
2967                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2968         if (ret < 0)
2969                 return ret;
2970         return 0;
2971 }
2972
2973 /**
2974  * Validate eCPRI item.
2975  *
2976  * @param[in] item
2977  *   Item specification.
2978  * @param[in] item_flags
2979  *   Bit-fields that holds the items detected until now.
2980  * @param[in] last_item
2981  *   Previous validated item in the pattern items.
2982  * @param[in] ether_type
2983  *   Type in the ethernet layer header (including dot1q).
2984  * @param[in] acc_mask
2985  *   Acceptable mask, if NULL default internal default mask
2986  *   will be used to check whether item fields are supported.
2987  * @param[out] error
2988  *   Pointer to error structure.
2989  *
2990  * @return
2991  *   0 on success, a negative errno value otherwise and rte_errno is set.
2992  */
2993 int
2994 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2995                               uint64_t item_flags,
2996                               uint64_t last_item,
2997                               uint16_t ether_type,
2998                               const struct rte_flow_item_ecpri *acc_mask,
2999                               struct rte_flow_error *error)
3000 {
3001         const struct rte_flow_item_ecpri *mask = item->mask;
3002         const struct rte_flow_item_ecpri nic_mask = {
3003                 .hdr = {
3004                         .common = {
3005                                 .u32 =
3006                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3007                                         .type = 0xFF,
3008                                         }).u32),
3009                         },
3010                         .dummy[0] = 0xFFFFFFFF,
3011                 },
3012         };
3013         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3014                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3015         struct rte_flow_item_ecpri mask_lo;
3016
3017         if (!(last_item & outer_l2_vlan) &&
3018             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3019                 return rte_flow_error_set(error, EINVAL,
3020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3021                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3022         if ((last_item & outer_l2_vlan) && ether_type &&
3023             ether_type != RTE_ETHER_TYPE_ECPRI)
3024                 return rte_flow_error_set(error, EINVAL,
3025                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3026                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3027         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3028                 return rte_flow_error_set(error, EINVAL,
3029                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3030                                           "eCPRI with tunnel is not supported right now");
3031         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3032                 return rte_flow_error_set(error, ENOTSUP,
3033                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3034                                           "multiple L3 layers not supported");
3035         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3036                 return rte_flow_error_set(error, EINVAL,
3037                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3038                                           "eCPRI cannot coexist with a TCP layer");
3039         /* In specification, eCPRI could be over UDP layer. */
3040         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3041                 return rte_flow_error_set(error, EINVAL,
3042                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3043                                           "eCPRI over UDP layer is not yet supported right now");
3044         /* Mask for type field in common header could be zero. */
3045         if (!mask)
3046                 mask = &rte_flow_item_ecpri_mask;
3047         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3048         /* Input mask is in big-endian format. */
3049         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3050                 return rte_flow_error_set(error, EINVAL,
3051                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3052                                           "partial mask is not supported for protocol");
3053         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3054                 return rte_flow_error_set(error, EINVAL,
3055                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3056                                           "message header mask must be after a type mask");
3057         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3058                                          acc_mask ? (const uint8_t *)acc_mask
3059                                                   : (const uint8_t *)&nic_mask,
3060                                          sizeof(struct rte_flow_item_ecpri),
3061                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3062 }
3063
3064 /**
3065  * Release resource related QUEUE/RSS action split.
3066  *
3067  * @param dev
3068  *   Pointer to Ethernet device.
3069  * @param flow
3070  *   Flow to release id's from.
3071  */
3072 static void
3073 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
3074                              struct rte_flow *flow)
3075 {
3076         struct mlx5_priv *priv = dev->data->dev_private;
3077         uint32_t handle_idx;
3078         struct mlx5_flow_handle *dev_handle;
3079
3080         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
3081                        handle_idx, dev_handle, next)
3082                 if (dev_handle->split_flow_id &&
3083                     !dev_handle->is_meter_flow_id)
3084                         mlx5_ipool_free(priv->sh->ipool
3085                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
3086                                         dev_handle->split_flow_id);
3087 }
3088
3089 static int
3090 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3091                    const struct rte_flow_attr *attr __rte_unused,
3092                    const struct rte_flow_item items[] __rte_unused,
3093                    const struct rte_flow_action actions[] __rte_unused,
3094                    bool external __rte_unused,
3095                    int hairpin __rte_unused,
3096                    struct rte_flow_error *error)
3097 {
3098         return rte_flow_error_set(error, ENOTSUP,
3099                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3100 }
3101
3102 static struct mlx5_flow *
3103 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3104                   const struct rte_flow_attr *attr __rte_unused,
3105                   const struct rte_flow_item items[] __rte_unused,
3106                   const struct rte_flow_action actions[] __rte_unused,
3107                   struct rte_flow_error *error)
3108 {
3109         rte_flow_error_set(error, ENOTSUP,
3110                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3111         return NULL;
3112 }
3113
3114 static int
3115 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3116                     struct mlx5_flow *dev_flow __rte_unused,
3117                     const struct rte_flow_attr *attr __rte_unused,
3118                     const struct rte_flow_item items[] __rte_unused,
3119                     const struct rte_flow_action actions[] __rte_unused,
3120                     struct rte_flow_error *error)
3121 {
3122         return rte_flow_error_set(error, ENOTSUP,
3123                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3124 }
3125
3126 static int
3127 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3128                 struct rte_flow *flow __rte_unused,
3129                 struct rte_flow_error *error)
3130 {
3131         return rte_flow_error_set(error, ENOTSUP,
3132                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3133 }
3134
3135 static void
3136 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3137                  struct rte_flow *flow __rte_unused)
3138 {
3139 }
3140
3141 static void
3142 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3143                   struct rte_flow *flow __rte_unused)
3144 {
3145 }
3146
3147 static int
3148 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3149                 struct rte_flow *flow __rte_unused,
3150                 const struct rte_flow_action *actions __rte_unused,
3151                 void *data __rte_unused,
3152                 struct rte_flow_error *error)
3153 {
3154         return rte_flow_error_set(error, ENOTSUP,
3155                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3156 }
3157
3158 static int
3159 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3160                       uint32_t domains __rte_unused,
3161                       uint32_t flags __rte_unused)
3162 {
3163         return 0;
3164 }
3165
3166 /* Void driver to protect from null pointer reference. */
3167 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3168         .validate = flow_null_validate,
3169         .prepare = flow_null_prepare,
3170         .translate = flow_null_translate,
3171         .apply = flow_null_apply,
3172         .remove = flow_null_remove,
3173         .destroy = flow_null_destroy,
3174         .query = flow_null_query,
3175         .sync_domain = flow_null_sync_domain,
3176 };
3177
3178 /**
3179  * Select flow driver type according to flow attributes and device
3180  * configuration.
3181  *
3182  * @param[in] dev
3183  *   Pointer to the dev structure.
3184  * @param[in] attr
3185  *   Pointer to the flow attributes.
3186  *
3187  * @return
3188  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3189  */
3190 static enum mlx5_flow_drv_type
3191 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3192 {
3193         struct mlx5_priv *priv = dev->data->dev_private;
3194         /* The OS can determine first a specific flow type (DV, VERBS) */
3195         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3196
3197         if (type != MLX5_FLOW_TYPE_MAX)
3198                 return type;
3199         /* If no OS specific type - continue with DV/VERBS selection */
3200         if (attr->transfer && priv->config.dv_esw_en)
3201                 type = MLX5_FLOW_TYPE_DV;
3202         if (!attr->transfer)
3203                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3204                                                  MLX5_FLOW_TYPE_VERBS;
3205         return type;
3206 }
3207
3208 #define flow_get_drv_ops(type) flow_drv_ops[type]
3209
3210 /**
3211  * Flow driver validation API. This abstracts calling driver specific functions.
3212  * The type of flow driver is determined according to flow attributes.
3213  *
3214  * @param[in] dev
3215  *   Pointer to the dev structure.
3216  * @param[in] attr
3217  *   Pointer to the flow attributes.
3218  * @param[in] items
3219  *   Pointer to the list of items.
3220  * @param[in] actions
3221  *   Pointer to the list of actions.
3222  * @param[in] external
3223  *   This flow rule is created by request external to PMD.
3224  * @param[in] hairpin
3225  *   Number of hairpin TX actions, 0 means classic flow.
3226  * @param[out] error
3227  *   Pointer to the error structure.
3228  *
3229  * @return
3230  *   0 on success, a negative errno value otherwise and rte_errno is set.
3231  */
3232 static inline int
3233 flow_drv_validate(struct rte_eth_dev *dev,
3234                   const struct rte_flow_attr *attr,
3235                   const struct rte_flow_item items[],
3236                   const struct rte_flow_action actions[],
3237                   bool external, int hairpin, struct rte_flow_error *error)
3238 {
3239         const struct mlx5_flow_driver_ops *fops;
3240         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3241
3242         fops = flow_get_drv_ops(type);
3243         return fops->validate(dev, attr, items, actions, external,
3244                               hairpin, error);
3245 }
3246
3247 /**
3248  * Flow driver preparation API. This abstracts calling driver specific
3249  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3250  * calculates the size of memory required for device flow, allocates the memory,
3251  * initializes the device flow and returns the pointer.
3252  *
3253  * @note
3254  *   This function initializes device flow structure such as dv or verbs in
3255  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3256  *   rest. For example, adding returning device flow to flow->dev_flow list and
3257  *   setting backward reference to the flow should be done out of this function.
3258  *   layers field is not filled either.
3259  *
3260  * @param[in] dev
3261  *   Pointer to the dev structure.
3262  * @param[in] attr
3263  *   Pointer to the flow attributes.
3264  * @param[in] items
3265  *   Pointer to the list of items.
3266  * @param[in] actions
3267  *   Pointer to the list of actions.
3268  * @param[in] flow_idx
3269  *   This memory pool index to the flow.
3270  * @param[out] error
3271  *   Pointer to the error structure.
3272  *
3273  * @return
3274  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3275  */
3276 static inline struct mlx5_flow *
3277 flow_drv_prepare(struct rte_eth_dev *dev,
3278                  const struct rte_flow *flow,
3279                  const struct rte_flow_attr *attr,
3280                  const struct rte_flow_item items[],
3281                  const struct rte_flow_action actions[],
3282                  uint32_t flow_idx,
3283                  struct rte_flow_error *error)
3284 {
3285         const struct mlx5_flow_driver_ops *fops;
3286         enum mlx5_flow_drv_type type = flow->drv_type;
3287         struct mlx5_flow *mlx5_flow = NULL;
3288
3289         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3290         fops = flow_get_drv_ops(type);
3291         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3292         if (mlx5_flow)
3293                 mlx5_flow->flow_idx = flow_idx;
3294         return mlx5_flow;
3295 }
3296
3297 /**
3298  * Flow driver translation API. This abstracts calling driver specific
3299  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3300  * translates a generic flow into a driver flow. flow_drv_prepare() must
3301  * precede.
3302  *
3303  * @note
3304  *   dev_flow->layers could be filled as a result of parsing during translation
3305  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3306  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3307  *   flow->actions could be overwritten even though all the expanded dev_flows
3308  *   have the same actions.
3309  *
3310  * @param[in] dev
3311  *   Pointer to the rte dev structure.
3312  * @param[in, out] dev_flow
3313  *   Pointer to the mlx5 flow.
3314  * @param[in] attr
3315  *   Pointer to the flow attributes.
3316  * @param[in] items
3317  *   Pointer to the list of items.
3318  * @param[in] actions
3319  *   Pointer to the list of actions.
3320  * @param[out] error
3321  *   Pointer to the error structure.
3322  *
3323  * @return
3324  *   0 on success, a negative errno value otherwise and rte_errno is set.
3325  */
3326 static inline int
3327 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3328                    const struct rte_flow_attr *attr,
3329                    const struct rte_flow_item items[],
3330                    const struct rte_flow_action actions[],
3331                    struct rte_flow_error *error)
3332 {
3333         const struct mlx5_flow_driver_ops *fops;
3334         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3335
3336         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3337         fops = flow_get_drv_ops(type);
3338         return fops->translate(dev, dev_flow, attr, items, actions, error);
3339 }
3340
3341 /**
3342  * Flow driver apply API. This abstracts calling driver specific functions.
3343  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3344  * translated driver flows on to device. flow_drv_translate() must precede.
3345  *
3346  * @param[in] dev
3347  *   Pointer to Ethernet device structure.
3348  * @param[in, out] flow
3349  *   Pointer to flow structure.
3350  * @param[out] error
3351  *   Pointer to error structure.
3352  *
3353  * @return
3354  *   0 on success, a negative errno value otherwise and rte_errno is set.
3355  */
3356 static inline int
3357 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3358                struct rte_flow_error *error)
3359 {
3360         const struct mlx5_flow_driver_ops *fops;
3361         enum mlx5_flow_drv_type type = flow->drv_type;
3362
3363         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3364         fops = flow_get_drv_ops(type);
3365         return fops->apply(dev, flow, error);
3366 }
3367
3368 /**
3369  * Flow driver destroy API. This abstracts calling driver specific functions.
3370  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3371  * on device and releases resources of the flow.
3372  *
3373  * @param[in] dev
3374  *   Pointer to Ethernet device.
3375  * @param[in, out] flow
3376  *   Pointer to flow structure.
3377  */
3378 static inline void
3379 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3380 {
3381         const struct mlx5_flow_driver_ops *fops;
3382         enum mlx5_flow_drv_type type = flow->drv_type;
3383
3384         flow_mreg_split_qrss_release(dev, flow);
3385         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3386         fops = flow_get_drv_ops(type);
3387         fops->destroy(dev, flow);
3388 }
3389
3390 /**
3391  * Flow driver find RSS policy tbl API. This abstracts calling driver
3392  * specific functions. Parent flow (rte_flow) should have driver
3393  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3394  *
3395  * @param[in] dev
3396  *   Pointer to Ethernet device.
3397  * @param[in, out] flow
3398  *   Pointer to flow structure.
3399  * @param[in] policy
3400  *   Pointer to meter policy table.
3401  * @param[in] rss_desc
3402  *   Pointer to rss_desc
3403  */
3404 static struct mlx5_flow_meter_sub_policy *
3405 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3406                 struct rte_flow *flow,
3407                 struct mlx5_flow_meter_policy *policy,
3408                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3409 {
3410         const struct mlx5_flow_driver_ops *fops;
3411         enum mlx5_flow_drv_type type = flow->drv_type;
3412
3413         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3414         fops = flow_get_drv_ops(type);
3415         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3416 }
3417
3418 /**
3419  * Get RSS action from the action list.
3420  *
3421  * @param[in] dev
3422  *   Pointer to Ethernet device.
3423  * @param[in] actions
3424  *   Pointer to the list of actions.
3425  * @param[in] flow
3426  *   Parent flow structure pointer.
3427  *
3428  * @return
3429  *   Pointer to the RSS action if exist, else return NULL.
3430  */
3431 static const struct rte_flow_action_rss*
3432 flow_get_rss_action(struct rte_eth_dev *dev,
3433                     const struct rte_flow_action actions[])
3434 {
3435         struct mlx5_priv *priv = dev->data->dev_private;
3436         const struct rte_flow_action_rss *rss = NULL;
3437
3438         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3439                 switch (actions->type) {
3440                 case RTE_FLOW_ACTION_TYPE_RSS:
3441                         rss = actions->conf;
3442                         break;
3443                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3444                 {
3445                         const struct rte_flow_action_sample *sample =
3446                                                                 actions->conf;
3447                         const struct rte_flow_action *act = sample->actions;
3448                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3449                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3450                                         rss = act->conf;
3451                         break;
3452                 }
3453                 case RTE_FLOW_ACTION_TYPE_METER:
3454                 {
3455                         uint32_t mtr_idx;
3456                         struct mlx5_flow_meter_info *fm;
3457                         struct mlx5_flow_meter_policy *policy;
3458                         const struct rte_flow_action_meter *mtr = actions->conf;
3459
3460                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3461                         if (fm) {
3462                                 policy = mlx5_flow_meter_policy_find(dev,
3463                                                 fm->policy_id, NULL);
3464                                 if (policy && policy->is_rss)
3465                                         rss =
3466                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3467                         }
3468                         break;
3469                 }
3470                 default:
3471                         break;
3472                 }
3473         }
3474         return rss;
3475 }
3476
3477 /**
3478  * Get ASO age action by index.
3479  *
3480  * @param[in] dev
3481  *   Pointer to the Ethernet device structure.
3482  * @param[in] age_idx
3483  *   Index to the ASO age action.
3484  *
3485  * @return
3486  *   The specified ASO age action.
3487  */
3488 struct mlx5_aso_age_action*
3489 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3490 {
3491         uint16_t pool_idx = age_idx & UINT16_MAX;
3492         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3493         struct mlx5_priv *priv = dev->data->dev_private;
3494         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3495         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3496
3497         return &pool->actions[offset - 1];
3498 }
3499
3500 /* maps indirect action to translated direct in some actions array */
3501 struct mlx5_translated_action_handle {
3502         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3503         int index; /**< Index in related array of rte_flow_action. */
3504 };
3505
3506 /**
3507  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3508  * direct action if translation possible.
3509  * This functionality used to run same execution path for both direct and
3510  * indirect actions on flow create. All necessary preparations for indirect
3511  * action handling should be performed on *handle* actions list returned
3512  * from this call.
3513  *
3514  * @param[in] dev
3515  *   Pointer to Ethernet device.
3516  * @param[in] actions
3517  *   List of actions to translate.
3518  * @param[out] handle
3519  *   List to store translated indirect action object handles.
3520  * @param[in, out] indir_n
3521  *   Size of *handle* array. On return should be updated with number of
3522  *   indirect actions retrieved from the *actions* list.
3523  * @param[out] translated_actions
3524  *   List of actions where all indirect actions were translated to direct
3525  *   if possible. NULL if no translation took place.
3526  * @param[out] error
3527  *   Pointer to the error structure.
3528  *
3529  * @return
3530  *   0 on success, a negative errno value otherwise and rte_errno is set.
3531  */
3532 static int
3533 flow_action_handles_translate(struct rte_eth_dev *dev,
3534                               const struct rte_flow_action actions[],
3535                               struct mlx5_translated_action_handle *handle,
3536                               int *indir_n,
3537                               struct rte_flow_action **translated_actions,
3538                               struct rte_flow_error *error)
3539 {
3540         struct mlx5_priv *priv = dev->data->dev_private;
3541         struct rte_flow_action *translated = NULL;
3542         size_t actions_size;
3543         int n;
3544         int copied_n = 0;
3545         struct mlx5_translated_action_handle *handle_end = NULL;
3546
3547         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3548                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3549                         continue;
3550                 if (copied_n == *indir_n) {
3551                         return rte_flow_error_set
3552                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3553                                  NULL, "too many shared actions");
3554                 }
3555                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3556                            sizeof(actions[n].conf));
3557                 handle[copied_n].index = n;
3558                 copied_n++;
3559         }
3560         n++;
3561         *indir_n = copied_n;
3562         if (!copied_n)
3563                 return 0;
3564         actions_size = sizeof(struct rte_flow_action) * n;
3565         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3566         if (!translated) {
3567                 rte_errno = ENOMEM;
3568                 return -ENOMEM;
3569         }
3570         memcpy(translated, actions, actions_size);
3571         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3572                 struct mlx5_shared_action_rss *shared_rss;
3573                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3574                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3575                 uint32_t idx = act_idx &
3576                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3577
3578                 switch (type) {
3579                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3580                         shared_rss = mlx5_ipool_get
3581                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3582                         translated[handle->index].type =
3583                                 RTE_FLOW_ACTION_TYPE_RSS;
3584                         translated[handle->index].conf =
3585                                 &shared_rss->origin;
3586                         break;
3587                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3588                         translated[handle->index].type =
3589                                                 (enum rte_flow_action_type)
3590                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3591                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3592                         break;
3593                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3594                         if (priv->sh->flow_hit_aso_en) {
3595                                 translated[handle->index].type =
3596                                         (enum rte_flow_action_type)
3597                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3598                                 translated[handle->index].conf =
3599                                                          (void *)(uintptr_t)idx;
3600                                 break;
3601                         }
3602                         /* Fall-through */
3603                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3604                         if (priv->sh->ct_aso_en) {
3605                                 translated[handle->index].type =
3606                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3607                                 translated[handle->index].conf =
3608                                                          (void *)(uintptr_t)idx;
3609                                 break;
3610                         }
3611                         /* Fall-through */
3612                 default:
3613                         mlx5_free(translated);
3614                         return rte_flow_error_set
3615                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3616                                  NULL, "invalid indirect action type");
3617                 }
3618         }
3619         *translated_actions = translated;
3620         return 0;
3621 }
3622
3623 /**
3624  * Get Shared RSS action from the action list.
3625  *
3626  * @param[in] dev
3627  *   Pointer to Ethernet device.
3628  * @param[in] shared
3629  *   Pointer to the list of actions.
3630  * @param[in] shared_n
3631  *   Actions list length.
3632  *
3633  * @return
3634  *   The MLX5 RSS action ID if exists, otherwise return 0.
3635  */
3636 static uint32_t
3637 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3638                            struct mlx5_translated_action_handle *handle,
3639                            int shared_n)
3640 {
3641         struct mlx5_translated_action_handle *handle_end;
3642         struct mlx5_priv *priv = dev->data->dev_private;
3643         struct mlx5_shared_action_rss *shared_rss;
3644
3645
3646         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3647                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3648                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3649                 uint32_t idx = act_idx &
3650                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3651                 switch (type) {
3652                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3653                         shared_rss = mlx5_ipool_get
3654                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3655                                                                            idx);
3656                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3657                                            __ATOMIC_RELAXED);
3658                         return idx;
3659                 default:
3660                         break;
3661                 }
3662         }
3663         return 0;
3664 }
3665
3666 static unsigned int
3667 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3668 {
3669         const struct rte_flow_item *item;
3670         unsigned int has_vlan = 0;
3671
3672         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3673                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3674                         has_vlan = 1;
3675                         break;
3676                 }
3677         }
3678         if (has_vlan)
3679                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3680                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3681         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3682                                MLX5_EXPANSION_ROOT_OUTER;
3683 }
3684
3685 /**
3686  *  Get layer flags from the prefix flow.
3687  *
3688  *  Some flows may be split to several subflows, the prefix subflow gets the
3689  *  match items and the suffix sub flow gets the actions.
3690  *  Some actions need the user defined match item flags to get the detail for
3691  *  the action.
3692  *  This function helps the suffix flow to get the item layer flags from prefix
3693  *  subflow.
3694  *
3695  * @param[in] dev_flow
3696  *   Pointer the created preifx subflow.
3697  *
3698  * @return
3699  *   The layers get from prefix subflow.
3700  */
3701 static inline uint64_t
3702 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3703 {
3704         uint64_t layers = 0;
3705
3706         /*
3707          * Layers bits could be localization, but usually the compiler will
3708          * help to do the optimization work for source code.
3709          * If no decap actions, use the layers directly.
3710          */
3711         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3712                 return dev_flow->handle->layers;
3713         /* Convert L3 layers with decap action. */
3714         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3715                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3716         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3717                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3718         /* Convert L4 layers with decap action.  */
3719         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3720                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3721         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3722                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3723         return layers;
3724 }
3725
3726 /**
3727  * Get metadata split action information.
3728  *
3729  * @param[in] actions
3730  *   Pointer to the list of actions.
3731  * @param[out] qrss
3732  *   Pointer to the return pointer.
3733  * @param[out] qrss_type
3734  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3735  *   if no QUEUE/RSS is found.
3736  * @param[out] encap_idx
3737  *   Pointer to the index of the encap action if exists, otherwise the last
3738  *   action index.
3739  *
3740  * @return
3741  *   Total number of actions.
3742  */
3743 static int
3744 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3745                                        const struct rte_flow_action **qrss,
3746                                        int *encap_idx)
3747 {
3748         const struct rte_flow_action_raw_encap *raw_encap;
3749         int actions_n = 0;
3750         int raw_decap_idx = -1;
3751
3752         *encap_idx = -1;
3753         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3754                 switch (actions->type) {
3755                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3756                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3757                         *encap_idx = actions_n;
3758                         break;
3759                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3760                         raw_decap_idx = actions_n;
3761                         break;
3762                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3763                         raw_encap = actions->conf;
3764                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3765                                 *encap_idx = raw_decap_idx != -1 ?
3766                                                       raw_decap_idx : actions_n;
3767                         break;
3768                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3769                 case RTE_FLOW_ACTION_TYPE_RSS:
3770                         *qrss = actions;
3771                         break;
3772                 default:
3773                         break;
3774                 }
3775                 actions_n++;
3776         }
3777         if (*encap_idx == -1)
3778                 *encap_idx = actions_n;
3779         /* Count RTE_FLOW_ACTION_TYPE_END. */
3780         return actions_n + 1;
3781 }
3782
3783 /**
3784  * Check if the action will change packet.
3785  *
3786  * @param dev
3787  *   Pointer to Ethernet device.
3788  * @param[in] type
3789  *   action type.
3790  *
3791  * @return
3792  *   true if action will change packet, false otherwise.
3793  */
3794 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3795                                           enum rte_flow_action_type type)
3796 {
3797         struct mlx5_priv *priv = dev->data->dev_private;
3798
3799         switch (type) {
3800         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3801         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3802         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3803         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3804         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3805         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3806         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3807         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3808         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3809         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3810         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3811         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3812         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3813         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3814         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3815         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3816         case RTE_FLOW_ACTION_TYPE_SET_META:
3817         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3818         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3819         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3820         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3821         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3822         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3823         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3824         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3825         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3826         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3827         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3828         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3829                 return true;
3830         case RTE_FLOW_ACTION_TYPE_FLAG:
3831         case RTE_FLOW_ACTION_TYPE_MARK:
3832                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3833                         return true;
3834                 else
3835                         return false;
3836         default:
3837                 return false;
3838         }
3839 }
3840
3841 /**
3842  * Check meter action from the action list.
3843  *
3844  * @param dev
3845  *   Pointer to Ethernet device.
3846  * @param[in] actions
3847  *   Pointer to the list of actions.
3848  * @param[out] has_mtr
3849  *   Pointer to the meter exist flag.
3850  * @param[out] has_modify
3851  *   Pointer to the flag showing there's packet change action.
3852  * @param[out] meter_id
3853  *   Pointer to the meter id.
3854  *
3855  * @return
3856  *   Total number of actions.
3857  */
3858 static int
3859 flow_check_meter_action(struct rte_eth_dev *dev,
3860                         const struct rte_flow_action actions[],
3861                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3862 {
3863         const struct rte_flow_action_meter *mtr = NULL;
3864         int actions_n = 0;
3865
3866         MLX5_ASSERT(has_mtr);
3867         *has_mtr = false;
3868         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3869                 switch (actions->type) {
3870                 case RTE_FLOW_ACTION_TYPE_METER:
3871                         mtr = actions->conf;
3872                         *meter_id = mtr->mtr_id;
3873                         *has_mtr = true;
3874                         break;
3875                 default:
3876                         break;
3877                 }
3878                 if (!*has_mtr)
3879                         *has_modify |= flow_check_modify_action_type(dev,
3880                                                                 actions->type);
3881                 actions_n++;
3882         }
3883         /* Count RTE_FLOW_ACTION_TYPE_END. */
3884         return actions_n + 1;
3885 }
3886
3887 /**
3888  * Check if the flow should be split due to hairpin.
3889  * The reason for the split is that in current HW we can't
3890  * support encap and push-vlan on Rx, so if a flow contains
3891  * these actions we move it to Tx.
3892  *
3893  * @param dev
3894  *   Pointer to Ethernet device.
3895  * @param[in] attr
3896  *   Flow rule attributes.
3897  * @param[in] actions
3898  *   Associated actions (list terminated by the END action).
3899  *
3900  * @return
3901  *   > 0 the number of actions and the flow should be split,
3902  *   0 when no split required.
3903  */
3904 static int
3905 flow_check_hairpin_split(struct rte_eth_dev *dev,
3906                          const struct rte_flow_attr *attr,
3907                          const struct rte_flow_action actions[])
3908 {
3909         int queue_action = 0;
3910         int action_n = 0;
3911         int split = 0;
3912         const struct rte_flow_action_queue *queue;
3913         const struct rte_flow_action_rss *rss;
3914         const struct rte_flow_action_raw_encap *raw_encap;
3915         const struct rte_eth_hairpin_conf *conf;
3916
3917         if (!attr->ingress)
3918                 return 0;
3919         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3920                 switch (actions->type) {
3921                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3922                         queue = actions->conf;
3923                         if (queue == NULL)
3924                                 return 0;
3925                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3926                         if (conf == NULL || conf->tx_explicit != 0)
3927                                 return 0;
3928                         queue_action = 1;
3929                         action_n++;
3930                         break;
3931                 case RTE_FLOW_ACTION_TYPE_RSS:
3932                         rss = actions->conf;
3933                         if (rss == NULL || rss->queue_num == 0)
3934                                 return 0;
3935                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3936                         if (conf == NULL || conf->tx_explicit != 0)
3937                                 return 0;
3938                         queue_action = 1;
3939                         action_n++;
3940                         break;
3941                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3942                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3943                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3944                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3945                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3946                         split++;
3947                         action_n++;
3948                         break;
3949                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3950                         raw_encap = actions->conf;
3951                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3952                                 split++;
3953                         action_n++;
3954                         break;
3955                 default:
3956                         action_n++;
3957                         break;
3958                 }
3959         }
3960         if (split && queue_action)
3961                 return action_n;
3962         return 0;
3963 }
3964
3965 /* Declare flow create/destroy prototype in advance. */
3966 static uint32_t
3967 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3968                  const struct rte_flow_attr *attr,
3969                  const struct rte_flow_item items[],
3970                  const struct rte_flow_action actions[],
3971                  bool external, struct rte_flow_error *error);
3972
3973 static void
3974 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3975                   uint32_t flow_idx);
3976
3977 int
3978 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
3979                       struct mlx5_hlist_entry *entry,
3980                       uint64_t key, void *cb_ctx __rte_unused)
3981 {
3982         struct mlx5_flow_mreg_copy_resource *mcp_res =
3983                 container_of(entry, typeof(*mcp_res), hlist_ent);
3984
3985         return mcp_res->mark_id != key;
3986 }
3987
3988 struct mlx5_hlist_entry *
3989 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3990                        void *cb_ctx)
3991 {
3992         struct rte_eth_dev *dev = list->ctx;
3993         struct mlx5_priv *priv = dev->data->dev_private;
3994         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3995         struct mlx5_flow_mreg_copy_resource *mcp_res;
3996         struct rte_flow_error *error = ctx->error;
3997         uint32_t idx = 0;
3998         int ret;
3999         uint32_t mark_id = key;
4000         struct rte_flow_attr attr = {
4001                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4002                 .ingress = 1,
4003         };
4004         struct mlx5_rte_flow_item_tag tag_spec = {
4005                 .data = mark_id,
4006         };
4007         struct rte_flow_item items[] = {
4008                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4009         };
4010         struct rte_flow_action_mark ftag = {
4011                 .id = mark_id,
4012         };
4013         struct mlx5_flow_action_copy_mreg cp_mreg = {
4014                 .dst = REG_B,
4015                 .src = REG_NON,
4016         };
4017         struct rte_flow_action_jump jump = {
4018                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4019         };
4020         struct rte_flow_action actions[] = {
4021                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4022         };
4023
4024         /* Fill the register fileds in the flow. */
4025         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4026         if (ret < 0)
4027                 return NULL;
4028         tag_spec.id = ret;
4029         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4030         if (ret < 0)
4031                 return NULL;
4032         cp_mreg.src = ret;
4033         /* Provide the full width of FLAG specific value. */
4034         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4035                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4036         /* Build a new flow. */
4037         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4038                 items[0] = (struct rte_flow_item){
4039                         .type = (enum rte_flow_item_type)
4040                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4041                         .spec = &tag_spec,
4042                 };
4043                 items[1] = (struct rte_flow_item){
4044                         .type = RTE_FLOW_ITEM_TYPE_END,
4045                 };
4046                 actions[0] = (struct rte_flow_action){
4047                         .type = (enum rte_flow_action_type)
4048                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4049                         .conf = &ftag,
4050                 };
4051                 actions[1] = (struct rte_flow_action){
4052                         .type = (enum rte_flow_action_type)
4053                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4054                         .conf = &cp_mreg,
4055                 };
4056                 actions[2] = (struct rte_flow_action){
4057                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4058                         .conf = &jump,
4059                 };
4060                 actions[3] = (struct rte_flow_action){
4061                         .type = RTE_FLOW_ACTION_TYPE_END,
4062                 };
4063         } else {
4064                 /* Default rule, wildcard match. */
4065                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4066                 items[0] = (struct rte_flow_item){
4067                         .type = RTE_FLOW_ITEM_TYPE_END,
4068                 };
4069                 actions[0] = (struct rte_flow_action){
4070                         .type = (enum rte_flow_action_type)
4071                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4072                         .conf = &cp_mreg,
4073                 };
4074                 actions[1] = (struct rte_flow_action){
4075                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4076                         .conf = &jump,
4077                 };
4078                 actions[2] = (struct rte_flow_action){
4079                         .type = RTE_FLOW_ACTION_TYPE_END,
4080                 };
4081         }
4082         /* Build a new entry. */
4083         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4084         if (!mcp_res) {
4085                 rte_errno = ENOMEM;
4086                 return NULL;
4087         }
4088         mcp_res->idx = idx;
4089         mcp_res->mark_id = mark_id;
4090         /*
4091          * The copy Flows are not included in any list. There
4092          * ones are referenced from other Flows and can not
4093          * be applied, removed, deleted in ardbitrary order
4094          * by list traversing.
4095          */
4096         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
4097                                          actions, false, error);
4098         if (!mcp_res->rix_flow) {
4099                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4100                 return NULL;
4101         }
4102         return &mcp_res->hlist_ent;
4103 }
4104
4105 /**
4106  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4107  *
4108  * As mark_id is unique, if there's already a registered flow for the mark_id,
4109  * return by increasing the reference counter of the resource. Otherwise, create
4110  * the resource (mcp_res) and flow.
4111  *
4112  * Flow looks like,
4113  *   - If ingress port is ANY and reg_c[1] is mark_id,
4114  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4115  *
4116  * For default flow (zero mark_id), flow is like,
4117  *   - If ingress port is ANY,
4118  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4119  *
4120  * @param dev
4121  *   Pointer to Ethernet device.
4122  * @param mark_id
4123  *   ID of MARK action, zero means default flow for META.
4124  * @param[out] error
4125  *   Perform verbose error reporting if not NULL.
4126  *
4127  * @return
4128  *   Associated resource on success, NULL otherwise and rte_errno is set.
4129  */
4130 static struct mlx5_flow_mreg_copy_resource *
4131 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4132                           struct rte_flow_error *error)
4133 {
4134         struct mlx5_priv *priv = dev->data->dev_private;
4135         struct mlx5_hlist_entry *entry;
4136         struct mlx5_flow_cb_ctx ctx = {
4137                 .dev = dev,
4138                 .error = error,
4139         };
4140
4141         /* Check if already registered. */
4142         MLX5_ASSERT(priv->mreg_cp_tbl);
4143         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4144         if (!entry)
4145                 return NULL;
4146         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4147                             hlist_ent);
4148 }
4149
4150 void
4151 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
4152 {
4153         struct mlx5_flow_mreg_copy_resource *mcp_res =
4154                 container_of(entry, typeof(*mcp_res), hlist_ent);
4155         struct rte_eth_dev *dev = list->ctx;
4156         struct mlx5_priv *priv = dev->data->dev_private;
4157
4158         MLX5_ASSERT(mcp_res->rix_flow);
4159         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
4160         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4161 }
4162
4163 /**
4164  * Release flow in RX_CP_TBL.
4165  *
4166  * @param dev
4167  *   Pointer to Ethernet device.
4168  * @flow
4169  *   Parent flow for wich copying is provided.
4170  */
4171 static void
4172 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4173                           struct rte_flow *flow)
4174 {
4175         struct mlx5_flow_mreg_copy_resource *mcp_res;
4176         struct mlx5_priv *priv = dev->data->dev_private;
4177
4178         if (!flow->rix_mreg_copy)
4179                 return;
4180         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4181                                  flow->rix_mreg_copy);
4182         if (!mcp_res || !priv->mreg_cp_tbl)
4183                 return;
4184         MLX5_ASSERT(mcp_res->rix_flow);
4185         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4186         flow->rix_mreg_copy = 0;
4187 }
4188
4189 /**
4190  * Remove the default copy action from RX_CP_TBL.
4191  *
4192  * This functions is called in the mlx5_dev_start(). No thread safe
4193  * is guaranteed.
4194  *
4195  * @param dev
4196  *   Pointer to Ethernet device.
4197  */
4198 static void
4199 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4200 {
4201         struct mlx5_hlist_entry *entry;
4202         struct mlx5_priv *priv = dev->data->dev_private;
4203
4204         /* Check if default flow is registered. */
4205         if (!priv->mreg_cp_tbl)
4206                 return;
4207         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
4208                                   MLX5_DEFAULT_COPY_ID, NULL);
4209         if (!entry)
4210                 return;
4211         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4212 }
4213
4214 /**
4215  * Add the default copy action in in RX_CP_TBL.
4216  *
4217  * This functions is called in the mlx5_dev_start(). No thread safe
4218  * is guaranteed.
4219  *
4220  * @param dev
4221  *   Pointer to Ethernet device.
4222  * @param[out] error
4223  *   Perform verbose error reporting if not NULL.
4224  *
4225  * @return
4226  *   0 for success, negative value otherwise and rte_errno is set.
4227  */
4228 static int
4229 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4230                                   struct rte_flow_error *error)
4231 {
4232         struct mlx5_priv *priv = dev->data->dev_private;
4233         struct mlx5_flow_mreg_copy_resource *mcp_res;
4234
4235         /* Check whether extensive metadata feature is engaged. */
4236         if (!priv->config.dv_flow_en ||
4237             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4238             !mlx5_flow_ext_mreg_supported(dev) ||
4239             !priv->sh->dv_regc0_mask)
4240                 return 0;
4241         /*
4242          * Add default mreg copy flow may be called multiple time, but
4243          * only be called once in stop. Avoid register it twice.
4244          */
4245         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
4246                 return 0;
4247         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4248         if (!mcp_res)
4249                 return -rte_errno;
4250         return 0;
4251 }
4252
4253 /**
4254  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4255  *
4256  * All the flow having Q/RSS action should be split by
4257  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4258  * performs the following,
4259  *   - CQE->flow_tag := reg_c[1] (MARK)
4260  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4261  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4262  * but there should be a flow per each MARK ID set by MARK action.
4263  *
4264  * For the aforementioned reason, if there's a MARK action in flow's action
4265  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4266  * the MARK ID to CQE's flow_tag like,
4267  *   - If reg_c[1] is mark_id,
4268  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4269  *
4270  * For SET_META action which stores value in reg_c[0], as the destination is
4271  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4272  * MARK ID means the default flow. The default flow looks like,
4273  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4274  *
4275  * @param dev
4276  *   Pointer to Ethernet device.
4277  * @param flow
4278  *   Pointer to flow structure.
4279  * @param[in] actions
4280  *   Pointer to the list of actions.
4281  * @param[out] error
4282  *   Perform verbose error reporting if not NULL.
4283  *
4284  * @return
4285  *   0 on success, negative value otherwise and rte_errno is set.
4286  */
4287 static int
4288 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4289                             struct rte_flow *flow,
4290                             const struct rte_flow_action *actions,
4291                             struct rte_flow_error *error)
4292 {
4293         struct mlx5_priv *priv = dev->data->dev_private;
4294         struct mlx5_dev_config *config = &priv->config;
4295         struct mlx5_flow_mreg_copy_resource *mcp_res;
4296         const struct rte_flow_action_mark *mark;
4297
4298         /* Check whether extensive metadata feature is engaged. */
4299         if (!config->dv_flow_en ||
4300             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4301             !mlx5_flow_ext_mreg_supported(dev) ||
4302             !priv->sh->dv_regc0_mask)
4303                 return 0;
4304         /* Find MARK action. */
4305         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4306                 switch (actions->type) {
4307                 case RTE_FLOW_ACTION_TYPE_FLAG:
4308                         mcp_res = flow_mreg_add_copy_action
4309                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4310                         if (!mcp_res)
4311                                 return -rte_errno;
4312                         flow->rix_mreg_copy = mcp_res->idx;
4313                         return 0;
4314                 case RTE_FLOW_ACTION_TYPE_MARK:
4315                         mark = (const struct rte_flow_action_mark *)
4316                                 actions->conf;
4317                         mcp_res =
4318                                 flow_mreg_add_copy_action(dev, mark->id, error);
4319                         if (!mcp_res)
4320                                 return -rte_errno;
4321                         flow->rix_mreg_copy = mcp_res->idx;
4322                         return 0;
4323                 default:
4324                         break;
4325                 }
4326         }
4327         return 0;
4328 }
4329
4330 #define MLX5_MAX_SPLIT_ACTIONS 24
4331 #define MLX5_MAX_SPLIT_ITEMS 24
4332
4333 /**
4334  * Split the hairpin flow.
4335  * Since HW can't support encap and push-vlan on Rx, we move these
4336  * actions to Tx.
4337  * If the count action is after the encap then we also
4338  * move the count action. in this case the count will also measure
4339  * the outer bytes.
4340  *
4341  * @param dev
4342  *   Pointer to Ethernet device.
4343  * @param[in] actions
4344  *   Associated actions (list terminated by the END action).
4345  * @param[out] actions_rx
4346  *   Rx flow actions.
4347  * @param[out] actions_tx
4348  *   Tx flow actions..
4349  * @param[out] pattern_tx
4350  *   The pattern items for the Tx flow.
4351  * @param[out] flow_id
4352  *   The flow ID connected to this flow.
4353  *
4354  * @return
4355  *   0 on success.
4356  */
4357 static int
4358 flow_hairpin_split(struct rte_eth_dev *dev,
4359                    const struct rte_flow_action actions[],
4360                    struct rte_flow_action actions_rx[],
4361                    struct rte_flow_action actions_tx[],
4362                    struct rte_flow_item pattern_tx[],
4363                    uint32_t flow_id)
4364 {
4365         const struct rte_flow_action_raw_encap *raw_encap;
4366         const struct rte_flow_action_raw_decap *raw_decap;
4367         struct mlx5_rte_flow_action_set_tag *set_tag;
4368         struct rte_flow_action *tag_action;
4369         struct mlx5_rte_flow_item_tag *tag_item;
4370         struct rte_flow_item *item;
4371         char *addr;
4372         int encap = 0;
4373
4374         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4375                 switch (actions->type) {
4376                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4377                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4378                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4379                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4380                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4381                         rte_memcpy(actions_tx, actions,
4382                                sizeof(struct rte_flow_action));
4383                         actions_tx++;
4384                         break;
4385                 case RTE_FLOW_ACTION_TYPE_COUNT:
4386                         if (encap) {
4387                                 rte_memcpy(actions_tx, actions,
4388                                            sizeof(struct rte_flow_action));
4389                                 actions_tx++;
4390                         } else {
4391                                 rte_memcpy(actions_rx, actions,
4392                                            sizeof(struct rte_flow_action));
4393                                 actions_rx++;
4394                         }
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4397                         raw_encap = actions->conf;
4398                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4399                                 memcpy(actions_tx, actions,
4400                                        sizeof(struct rte_flow_action));
4401                                 actions_tx++;
4402                                 encap = 1;
4403                         } else {
4404                                 rte_memcpy(actions_rx, actions,
4405                                            sizeof(struct rte_flow_action));
4406                                 actions_rx++;
4407                         }
4408                         break;
4409                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4410                         raw_decap = actions->conf;
4411                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4412                                 memcpy(actions_tx, actions,
4413                                        sizeof(struct rte_flow_action));
4414                                 actions_tx++;
4415                         } else {
4416                                 rte_memcpy(actions_rx, actions,
4417                                            sizeof(struct rte_flow_action));
4418                                 actions_rx++;
4419                         }
4420                         break;
4421                 default:
4422                         rte_memcpy(actions_rx, actions,
4423                                    sizeof(struct rte_flow_action));
4424                         actions_rx++;
4425                         break;
4426                 }
4427         }
4428         /* Add set meta action and end action for the Rx flow. */
4429         tag_action = actions_rx;
4430         tag_action->type = (enum rte_flow_action_type)
4431                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4432         actions_rx++;
4433         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4434         actions_rx++;
4435         set_tag = (void *)actions_rx;
4436         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4437                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4438                 .data = flow_id,
4439         };
4440         MLX5_ASSERT(set_tag->id > REG_NON);
4441         tag_action->conf = set_tag;
4442         /* Create Tx item list. */
4443         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4444         addr = (void *)&pattern_tx[2];
4445         item = pattern_tx;
4446         item->type = (enum rte_flow_item_type)
4447                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4448         tag_item = (void *)addr;
4449         tag_item->data = flow_id;
4450         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4451         MLX5_ASSERT(set_tag->id > REG_NON);
4452         item->spec = tag_item;
4453         addr += sizeof(struct mlx5_rte_flow_item_tag);
4454         tag_item = (void *)addr;
4455         tag_item->data = UINT32_MAX;
4456         tag_item->id = UINT16_MAX;
4457         item->mask = tag_item;
4458         item->last = NULL;
4459         item++;
4460         item->type = RTE_FLOW_ITEM_TYPE_END;
4461         return 0;
4462 }
4463
4464 /**
4465  * The last stage of splitting chain, just creates the subflow
4466  * without any modification.
4467  *
4468  * @param[in] dev
4469  *   Pointer to Ethernet device.
4470  * @param[in] flow
4471  *   Parent flow structure pointer.
4472  * @param[in, out] sub_flow
4473  *   Pointer to return the created subflow, may be NULL.
4474  * @param[in] attr
4475  *   Flow rule attributes.
4476  * @param[in] items
4477  *   Pattern specification (list terminated by the END pattern item).
4478  * @param[in] actions
4479  *   Associated actions (list terminated by the END action).
4480  * @param[in] flow_split_info
4481  *   Pointer to flow split info structure.
4482  * @param[out] error
4483  *   Perform verbose error reporting if not NULL.
4484  * @return
4485  *   0 on success, negative value otherwise
4486  */
4487 static int
4488 flow_create_split_inner(struct rte_eth_dev *dev,
4489                         struct rte_flow *flow,
4490                         struct mlx5_flow **sub_flow,
4491                         const struct rte_flow_attr *attr,
4492                         const struct rte_flow_item items[],
4493                         const struct rte_flow_action actions[],
4494                         struct mlx5_flow_split_info *flow_split_info,
4495                         struct rte_flow_error *error)
4496 {
4497         struct mlx5_flow *dev_flow;
4498
4499         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4500                                     flow_split_info->flow_idx, error);
4501         if (!dev_flow)
4502                 return -rte_errno;
4503         dev_flow->flow = flow;
4504         dev_flow->external = flow_split_info->external;
4505         dev_flow->skip_scale = flow_split_info->skip_scale;
4506         /* Subflow object was created, we must include one in the list. */
4507         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4508                       dev_flow->handle, next);
4509         /*
4510          * If dev_flow is as one of the suffix flow, some actions in suffix
4511          * flow may need some user defined item layer flags, and pass the
4512          * Metadate rxq mark flag to suffix flow as well.
4513          */
4514         if (flow_split_info->prefix_layers)
4515                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4516         if (flow_split_info->prefix_mark)
4517                 dev_flow->handle->mark = 1;
4518         if (sub_flow)
4519                 *sub_flow = dev_flow;
4520 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4521         dev_flow->dv.table_id = flow_split_info->table_id;
4522 #endif
4523         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4524 }
4525
4526 /**
4527  * Get the sub policy of a meter.
4528  *
4529  * @param[in] dev
4530  *   Pointer to Ethernet device.
4531  * @param[in] flow
4532  *   Parent flow structure pointer.
4533  * @param[in] policy_id;
4534  *   Meter Policy id.
4535  * @param[in] attr
4536  *   Flow rule attributes.
4537  * @param[in] items
4538  *   Pattern specification (list terminated by the END pattern item).
4539  * @param[out] error
4540  *   Perform verbose error reporting if not NULL.
4541  *
4542  * @return
4543  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4544  */
4545 static struct mlx5_flow_meter_sub_policy *
4546 get_meter_sub_policy(struct rte_eth_dev *dev,
4547                      struct rte_flow *flow,
4548                      uint32_t policy_id,
4549                      const struct rte_flow_attr *attr,
4550                      const struct rte_flow_item items[],
4551                      struct rte_flow_error *error)
4552 {
4553         struct mlx5_flow_meter_policy *policy;
4554         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4555
4556         policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL);
4557         if (!policy) {
4558                 rte_flow_error_set(error, EINVAL,
4559                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4560                                    "Failed to find Meter Policy.");
4561                 goto exit;
4562         }
4563         if (policy->is_rss) {
4564                 struct mlx5_flow_workspace *wks =
4565                                 mlx5_flow_get_thread_workspace();
4566                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4567                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4568                 uint32_t i;
4569
4570                 MLX5_ASSERT(wks);
4571                 /**
4572                  * This is a tmp dev_flow,
4573                  * no need to register any matcher for it in translate.
4574                  */
4575                 wks->skip_matcher_reg = 1;
4576                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4577                         struct mlx5_flow dev_flow = {0};
4578                         struct mlx5_flow_handle dev_handle = { {0} };
4579                         const void *rss_act = policy->act_cnt[i].rss->conf;
4580                         struct rte_flow_action rss_actions[2] = {
4581                                 [0] = {
4582                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4583                                         .conf = rss_act
4584                                 },
4585                                 [1] = {
4586                                         .type = RTE_FLOW_ACTION_TYPE_END,
4587                                         .conf = NULL
4588                                 }
4589                         };
4590
4591                         dev_flow.handle = &dev_handle;
4592                         dev_flow.ingress = attr->ingress;
4593                         dev_flow.flow = flow;
4594                         dev_flow.external = 0;
4595 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4596                         dev_flow.dv.transfer = attr->transfer;
4597 #endif
4598                         /* Translate RSS action to get rss hash fields. */
4599                         if (flow_drv_translate(dev, &dev_flow, attr,
4600                                                 items, rss_actions, error))
4601                                 goto exit;
4602                         rss_desc_v[i] = wks->rss_desc;
4603                         rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4604                         rss_desc_v[i].hash_fields = dev_flow.hash_fields;
4605                         rss_desc_v[i].queue_num = rss_desc_v[i].hash_fields ?
4606                                                   rss_desc_v[i].queue_num : 1;
4607                         rss_desc[i] = &rss_desc_v[i];
4608                 }
4609                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4610                                                 flow, policy, rss_desc);
4611         } else {
4612                 enum mlx5_meter_domain mtr_domain =
4613                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4614                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4615                                         MLX5_MTR_DOMAIN_INGRESS;
4616                 sub_policy = policy->sub_policys[mtr_domain][0];
4617         }
4618         if (!sub_policy) {
4619                 rte_flow_error_set(error, EINVAL,
4620                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4621                         "Failed to get meter sub-policy.");
4622                 goto exit;
4623         }
4624 exit:
4625         return sub_policy;
4626 }
4627
4628 /**
4629  * Split the meter flow.
4630  *
4631  * As meter flow will split to three sub flow, other than meter
4632  * action, the other actions make sense to only meter accepts
4633  * the packet. If it need to be dropped, no other additional
4634  * actions should be take.
4635  *
4636  * One kind of special action which decapsulates the L3 tunnel
4637  * header will be in the prefix sub flow, as not to take the
4638  * L3 tunnel header into account.
4639  *
4640  * @param[in] dev
4641  *   Pointer to Ethernet device.
4642  * @param[in] flow
4643  *   Parent flow structure pointer.
4644  * @param[in] fm
4645  *   Pointer to flow meter structure.
4646  * @param[in] attr
4647  *   Flow rule attributes.
4648  * @param[in] items
4649  *   Pattern specification (list terminated by the END pattern item).
4650  * @param[out] sfx_items
4651  *   Suffix flow match items (list terminated by the END pattern item).
4652  * @param[in] actions
4653  *   Associated actions (list terminated by the END action).
4654  * @param[out] actions_sfx
4655  *   Suffix flow actions.
4656  * @param[out] actions_pre
4657  *   Prefix flow actions.
4658  * @param[out] mtr_flow_id
4659  *   Pointer to meter flow id.
4660  * @param[out] error
4661  *   Perform verbose error reporting if not NULL.
4662  *
4663  * @return
4664  *   0 on success, a negative errno value otherwise and rte_errno is set.
4665  */
4666 static int
4667 flow_meter_split_prep(struct rte_eth_dev *dev,
4668                       struct rte_flow *flow,
4669                       struct mlx5_flow_meter_info *fm,
4670                       const struct rte_flow_attr *attr,
4671                       const struct rte_flow_item items[],
4672                       struct rte_flow_item sfx_items[],
4673                       const struct rte_flow_action actions[],
4674                       struct rte_flow_action actions_sfx[],
4675                       struct rte_flow_action actions_pre[],
4676                       uint32_t *mtr_flow_id,
4677                       struct rte_flow_error *error)
4678 {
4679         struct mlx5_priv *priv = dev->data->dev_private;
4680         struct rte_flow_action *tag_action = NULL;
4681         struct rte_flow_item *tag_item;
4682         struct mlx5_rte_flow_action_set_tag *set_tag;
4683         const struct rte_flow_action_raw_encap *raw_encap;
4684         const struct rte_flow_action_raw_decap *raw_decap;
4685         struct mlx5_rte_flow_item_tag *tag_item_spec;
4686         struct mlx5_rte_flow_item_tag *tag_item_mask;
4687         uint32_t tag_id = 0;
4688         bool copy_vlan = false;
4689         struct rte_flow_action *hw_mtr_action;
4690         struct rte_flow_action *action_pre_head = NULL;
4691         bool mtr_first = priv->sh->meter_aso_en &&
4692                         (attr->egress ||
4693                         (attr->transfer && priv->representor_id != UINT16_MAX));
4694         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4695         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4696                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4697         uint32_t flow_id = 0;
4698         uint32_t flow_id_reversed = 0;
4699         uint8_t flow_id_bits = 0;
4700         int shift;
4701
4702         /* For ASO meter, meter must be before tag in TX direction. */
4703         if (mtr_first) {
4704                 action_pre_head = actions_pre++;
4705                 /* Leave space for tag action. */
4706                 tag_action = actions_pre++;
4707         }
4708         /* Prepare the actions for prefix and suffix flow. */
4709         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4710                 struct rte_flow_action *action_cur = NULL;
4711
4712                 switch (actions->type) {
4713                 case RTE_FLOW_ACTION_TYPE_METER:
4714                         if (mtr_first) {
4715                                 action_cur = action_pre_head;
4716                         } else {
4717                                 /* Leave space for tag action. */
4718                                 tag_action = actions_pre++;
4719                                 action_cur = actions_pre++;
4720                         }
4721                         break;
4722                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4723                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4724                         action_cur = actions_pre++;
4725                         break;
4726                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4727                         raw_encap = actions->conf;
4728                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4729                                 action_cur = actions_pre++;
4730                         break;
4731                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4732                         raw_decap = actions->conf;
4733                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4734                                 action_cur = actions_pre++;
4735                         break;
4736                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4737                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4738                         copy_vlan = true;
4739                         break;
4740                 default:
4741                         break;
4742                 }
4743                 if (!action_cur)
4744                         action_cur = (fm->def_policy) ?
4745                                         actions_sfx++ : actions_pre++;
4746                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4747         }
4748         /* Add end action to the actions. */
4749         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4750         if (priv->sh->meter_aso_en) {
4751                 /**
4752                  * For ASO meter, need to add an extra jump action explicitly,
4753                  * to jump from meter to policer table.
4754                  */
4755                 struct mlx5_flow_meter_sub_policy *sub_policy;
4756                 struct mlx5_flow_tbl_data_entry *tbl_data;
4757
4758                 if (!fm->def_policy) {
4759                         sub_policy = get_meter_sub_policy(dev, flow,
4760                                                           fm->policy_id, attr,
4761                                                           items, error);
4762                         if (!sub_policy)
4763                                 return -rte_errno;
4764                 } else {
4765                         enum mlx5_meter_domain mtr_domain =
4766                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4767                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4768                                         MLX5_MTR_DOMAIN_INGRESS;
4769
4770                         sub_policy =
4771                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
4772                 }
4773                 tbl_data = container_of(sub_policy->tbl_rsc,
4774                                         struct mlx5_flow_tbl_data_entry, tbl);
4775                 hw_mtr_action = actions_pre++;
4776                 hw_mtr_action->type = (enum rte_flow_action_type)
4777                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
4778                 hw_mtr_action->conf = tbl_data->jump.action;
4779         }
4780         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4781         actions_pre++;
4782         if (!tag_action)
4783                 return rte_flow_error_set(error, ENOMEM,
4784                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4785                                         "No tag action space.");
4786         if (!mtr_flow_id) {
4787                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
4788                 goto exit;
4789         }
4790         /* Only default-policy Meter creates mtr flow id. */
4791         if (fm->def_policy) {
4792                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
4793                 if (!tag_id)
4794                         return rte_flow_error_set(error, ENOMEM,
4795                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4796                                         "Failed to allocate meter flow id.");
4797                 flow_id = tag_id - 1;
4798                 flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id);
4799                 flow_id_bits = flow_id_bits ? flow_id_bits : 1;
4800                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
4801                     mtr_reg_bits) {
4802                         mlx5_ipool_free(fm->flow_ipool, tag_id);
4803                         return rte_flow_error_set(error, EINVAL,
4804                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4805                                         "Meter flow id exceeds max limit.");
4806                 }
4807                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
4808                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
4809         }
4810         /* Prepare the suffix subflow items. */
4811         tag_item = sfx_items++;
4812         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4813                 int item_type = items->type;
4814
4815                 switch (item_type) {
4816                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4817                         memcpy(sfx_items, items, sizeof(*sfx_items));
4818                         sfx_items++;
4819                         break;
4820                 case RTE_FLOW_ITEM_TYPE_VLAN:
4821                         if (copy_vlan) {
4822                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4823                                 /*
4824                                  * Convert to internal match item, it is used
4825                                  * for vlan push and set vid.
4826                                  */
4827                                 sfx_items->type = (enum rte_flow_item_type)
4828                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4829                                 sfx_items++;
4830                         }
4831                         break;
4832                 default:
4833                         break;
4834                 }
4835         }
4836         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4837         sfx_items++;
4838         /* Build tag actions and items for meter_id/meter flow_id. */
4839         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
4840         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4841         tag_item_mask = tag_item_spec + 1;
4842         /* Both flow_id and meter_id share the same register. */
4843         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4844                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
4845                                                             0, error),
4846                 .offset = mtr_id_offset,
4847                 .length = mtr_reg_bits,
4848                 .data = flow->meter,
4849         };
4850         /*
4851          * The color Reg bits used by flow_id are growing from
4852          * msb to lsb, so must do bit reverse for flow_id val in RegC.
4853          */
4854         for (shift = 0; shift < flow_id_bits; shift++)
4855                 flow_id_reversed = (flow_id_reversed << 1) |
4856                                 ((flow_id >> shift) & 0x1);
4857         set_tag->data |=
4858                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
4859         tag_item_spec->id = set_tag->id;
4860         tag_item_spec->data = set_tag->data << mtr_id_offset;
4861         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
4862         tag_action->type = (enum rte_flow_action_type)
4863                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4864         tag_action->conf = set_tag;
4865         tag_item->type = (enum rte_flow_item_type)
4866                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4867         tag_item->spec = tag_item_spec;
4868         tag_item->last = NULL;
4869         tag_item->mask = tag_item_mask;
4870 exit:
4871         if (mtr_flow_id)
4872                 *mtr_flow_id = tag_id;
4873         return 0;
4874 }
4875
4876 /**
4877  * Split action list having QUEUE/RSS for metadata register copy.
4878  *
4879  * Once Q/RSS action is detected in user's action list, the flow action
4880  * should be split in order to copy metadata registers, which will happen in
4881  * RX_CP_TBL like,
4882  *   - CQE->flow_tag := reg_c[1] (MARK)
4883  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4884  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4885  * This is because the last action of each flow must be a terminal action
4886  * (QUEUE, RSS or DROP).
4887  *
4888  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4889  * stored and kept in the mlx5_flow structure per each sub_flow.
4890  *
4891  * The Q/RSS action is replaced with,
4892  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4893  * And the following JUMP action is added at the end,
4894  *   - JUMP, to RX_CP_TBL.
4895  *
4896  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4897  * flow_create_split_metadata() routine. The flow will look like,
4898  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4899  *
4900  * @param dev
4901  *   Pointer to Ethernet device.
4902  * @param[out] split_actions
4903  *   Pointer to store split actions to jump to CP_TBL.
4904  * @param[in] actions
4905  *   Pointer to the list of original flow actions.
4906  * @param[in] qrss
4907  *   Pointer to the Q/RSS action.
4908  * @param[in] actions_n
4909  *   Number of original actions.
4910  * @param[out] error
4911  *   Perform verbose error reporting if not NULL.
4912  *
4913  * @return
4914  *   non-zero unique flow_id on success, otherwise 0 and
4915  *   error/rte_error are set.
4916  */
4917 static uint32_t
4918 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4919                           struct rte_flow_action *split_actions,
4920                           const struct rte_flow_action *actions,
4921                           const struct rte_flow_action *qrss,
4922                           int actions_n, struct rte_flow_error *error)
4923 {
4924         struct mlx5_priv *priv = dev->data->dev_private;
4925         struct mlx5_rte_flow_action_set_tag *set_tag;
4926         struct rte_flow_action_jump *jump;
4927         const int qrss_idx = qrss - actions;
4928         uint32_t flow_id = 0;
4929         int ret = 0;
4930
4931         /*
4932          * Given actions will be split
4933          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4934          * - Add jump to mreg CP_TBL.
4935          * As a result, there will be one more action.
4936          */
4937         ++actions_n;
4938         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4939         set_tag = (void *)(split_actions + actions_n);
4940         /*
4941          * If tag action is not set to void(it means we are not the meter
4942          * suffix flow), add the tag action. Since meter suffix flow already
4943          * has the tag added.
4944          */
4945         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4946                 /*
4947                  * Allocate the new subflow ID. This one is unique within
4948                  * device and not shared with representors. Otherwise,
4949                  * we would have to resolve multi-thread access synch
4950                  * issue. Each flow on the shared device is appended
4951                  * with source vport identifier, so the resulting
4952                  * flows will be unique in the shared (by master and
4953                  * representors) domain even if they have coinciding
4954                  * IDs.
4955                  */
4956                 mlx5_ipool_malloc(priv->sh->ipool
4957                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4958                 if (!flow_id)
4959                         return rte_flow_error_set(error, ENOMEM,
4960                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4961                                                   NULL, "can't allocate id "
4962                                                   "for split Q/RSS subflow");
4963                 /* Internal SET_TAG action to set flow ID. */
4964                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4965                         .data = flow_id,
4966                 };
4967                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4968                 if (ret < 0)
4969                         return ret;
4970                 set_tag->id = ret;
4971                 /* Construct new actions array. */
4972                 /* Replace QUEUE/RSS action. */
4973                 split_actions[qrss_idx] = (struct rte_flow_action){
4974                         .type = (enum rte_flow_action_type)
4975                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4976                         .conf = set_tag,
4977                 };
4978         }
4979         /* JUMP action to jump to mreg copy table (CP_TBL). */
4980         jump = (void *)(set_tag + 1);
4981         *jump = (struct rte_flow_action_jump){
4982                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4983         };
4984         split_actions[actions_n - 2] = (struct rte_flow_action){
4985                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4986                 .conf = jump,
4987         };
4988         split_actions[actions_n - 1] = (struct rte_flow_action){
4989                 .type = RTE_FLOW_ACTION_TYPE_END,
4990         };
4991         return flow_id;
4992 }
4993
4994 /**
4995  * Extend the given action list for Tx metadata copy.
4996  *
4997  * Copy the given action list to the ext_actions and add flow metadata register
4998  * copy action in order to copy reg_a set by WQE to reg_c[0].
4999  *
5000  * @param[out] ext_actions
5001  *   Pointer to the extended action list.
5002  * @param[in] actions
5003  *   Pointer to the list of actions.
5004  * @param[in] actions_n
5005  *   Number of actions in the list.
5006  * @param[out] error
5007  *   Perform verbose error reporting if not NULL.
5008  * @param[in] encap_idx
5009  *   The encap action inndex.
5010  *
5011  * @return
5012  *   0 on success, negative value otherwise
5013  */
5014 static int
5015 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5016                        struct rte_flow_action *ext_actions,
5017                        const struct rte_flow_action *actions,
5018                        int actions_n, struct rte_flow_error *error,
5019                        int encap_idx)
5020 {
5021         struct mlx5_flow_action_copy_mreg *cp_mreg =
5022                 (struct mlx5_flow_action_copy_mreg *)
5023                         (ext_actions + actions_n + 1);
5024         int ret;
5025
5026         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5027         if (ret < 0)
5028                 return ret;
5029         cp_mreg->dst = ret;
5030         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5031         if (ret < 0)
5032                 return ret;
5033         cp_mreg->src = ret;
5034         if (encap_idx != 0)
5035                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5036         if (encap_idx == actions_n - 1) {
5037                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5038                         .type = (enum rte_flow_action_type)
5039                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5040                         .conf = cp_mreg,
5041                 };
5042                 ext_actions[actions_n] = (struct rte_flow_action){
5043                         .type = RTE_FLOW_ACTION_TYPE_END,
5044                 };
5045         } else {
5046                 ext_actions[encap_idx] = (struct rte_flow_action){
5047                         .type = (enum rte_flow_action_type)
5048                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5049                         .conf = cp_mreg,
5050                 };
5051                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5052                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5053         }
5054         return 0;
5055 }
5056
5057 /**
5058  * Check the match action from the action list.
5059  *
5060  * @param[in] actions
5061  *   Pointer to the list of actions.
5062  * @param[in] attr
5063  *   Flow rule attributes.
5064  * @param[in] action
5065  *   The action to be check if exist.
5066  * @param[out] match_action_pos
5067  *   Pointer to the position of the matched action if exists, otherwise is -1.
5068  * @param[out] qrss_action_pos
5069  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5070  * @param[out] modify_after_mirror
5071  *   Pointer to the flag of modify action after FDB mirroring.
5072  *
5073  * @return
5074  *   > 0 the total number of actions.
5075  *   0 if not found match action in action list.
5076  */
5077 static int
5078 flow_check_match_action(const struct rte_flow_action actions[],
5079                         const struct rte_flow_attr *attr,
5080                         enum rte_flow_action_type action,
5081                         int *match_action_pos, int *qrss_action_pos,
5082                         int *modify_after_mirror)
5083 {
5084         const struct rte_flow_action_sample *sample;
5085         int actions_n = 0;
5086         uint32_t ratio = 0;
5087         int sub_type = 0;
5088         int flag = 0;
5089         int fdb_mirror = 0;
5090
5091         *match_action_pos = -1;
5092         *qrss_action_pos = -1;
5093         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5094                 if (actions->type == action) {
5095                         flag = 1;
5096                         *match_action_pos = actions_n;
5097                 }
5098                 switch (actions->type) {
5099                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5100                 case RTE_FLOW_ACTION_TYPE_RSS:
5101                         *qrss_action_pos = actions_n;
5102                         break;
5103                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5104                         sample = actions->conf;
5105                         ratio = sample->ratio;
5106                         sub_type = ((const struct rte_flow_action *)
5107                                         (sample->actions))->type;
5108                         if (ratio == 1 && attr->transfer)
5109                                 fdb_mirror = 1;
5110                         break;
5111                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5112                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5113                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5114                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5115                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5116                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5117                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5118                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5119                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5120                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5121                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5122                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5123                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5124                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5125                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5126                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5127                 case RTE_FLOW_ACTION_TYPE_FLAG:
5128                 case RTE_FLOW_ACTION_TYPE_MARK:
5129                 case RTE_FLOW_ACTION_TYPE_SET_META:
5130                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5131                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5132                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5133                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5134                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5135                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5136                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5137                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5138                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5139                         if (fdb_mirror)
5140                                 *modify_after_mirror = 1;
5141                         break;
5142                 default:
5143                         break;
5144                 }
5145                 actions_n++;
5146         }
5147         if (flag && fdb_mirror && !*modify_after_mirror) {
5148                 /* FDB mirroring uses the destination array to implement
5149                  * instead of FLOW_SAMPLER object.
5150                  */
5151                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5152                         flag = 0;
5153         }
5154         /* Count RTE_FLOW_ACTION_TYPE_END. */
5155         return flag ? actions_n + 1 : 0;
5156 }
5157
5158 #define SAMPLE_SUFFIX_ITEM 2
5159
5160 /**
5161  * Split the sample flow.
5162  *
5163  * As sample flow will split to two sub flow, sample flow with
5164  * sample action, the other actions will move to new suffix flow.
5165  *
5166  * Also add unique tag id with tag action in the sample flow,
5167  * the same tag id will be as match in the suffix flow.
5168  *
5169  * @param dev
5170  *   Pointer to Ethernet device.
5171  * @param[in] add_tag
5172  *   Add extra tag action flag.
5173  * @param[out] sfx_items
5174  *   Suffix flow match items (list terminated by the END pattern item).
5175  * @param[in] actions
5176  *   Associated actions (list terminated by the END action).
5177  * @param[out] actions_sfx
5178  *   Suffix flow actions.
5179  * @param[out] actions_pre
5180  *   Prefix flow actions.
5181  * @param[in] actions_n
5182  *  The total number of actions.
5183  * @param[in] sample_action_pos
5184  *   The sample action position.
5185  * @param[in] qrss_action_pos
5186  *   The Queue/RSS action position.
5187  * @param[in] jump_table
5188  *   Add extra jump action flag.
5189  * @param[out] error
5190  *   Perform verbose error reporting if not NULL.
5191  *
5192  * @return
5193  *   0 on success, or unique flow_id, a negative errno value
5194  *   otherwise and rte_errno is set.
5195  */
5196 static int
5197 flow_sample_split_prep(struct rte_eth_dev *dev,
5198                        int add_tag,
5199                        struct rte_flow_item sfx_items[],
5200                        const struct rte_flow_action actions[],
5201                        struct rte_flow_action actions_sfx[],
5202                        struct rte_flow_action actions_pre[],
5203                        int actions_n,
5204                        int sample_action_pos,
5205                        int qrss_action_pos,
5206                        int jump_table,
5207                        struct rte_flow_error *error)
5208 {
5209         struct mlx5_priv *priv = dev->data->dev_private;
5210         struct mlx5_rte_flow_action_set_tag *set_tag;
5211         struct mlx5_rte_flow_item_tag *tag_spec;
5212         struct mlx5_rte_flow_item_tag *tag_mask;
5213         struct rte_flow_action_jump *jump_action;
5214         uint32_t tag_id = 0;
5215         int index;
5216         int append_index = 0;
5217         int ret;
5218
5219         if (sample_action_pos < 0)
5220                 return rte_flow_error_set(error, EINVAL,
5221                                           RTE_FLOW_ERROR_TYPE_ACTION,
5222                                           NULL, "invalid position of sample "
5223                                           "action in list");
5224         /* Prepare the actions for prefix and suffix flow. */
5225         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5226                 index = qrss_action_pos;
5227                 /* Put the preceding the Queue/RSS action into prefix flow. */
5228                 if (index != 0)
5229                         memcpy(actions_pre, actions,
5230                                sizeof(struct rte_flow_action) * index);
5231                 /* Put others preceding the sample action into prefix flow. */
5232                 if (sample_action_pos > index + 1)
5233                         memcpy(actions_pre + index, actions + index + 1,
5234                                sizeof(struct rte_flow_action) *
5235                                (sample_action_pos - index - 1));
5236                 index = sample_action_pos - 1;
5237                 /* Put Queue/RSS action into Suffix flow. */
5238                 memcpy(actions_sfx, actions + qrss_action_pos,
5239                        sizeof(struct rte_flow_action));
5240                 actions_sfx++;
5241         } else {
5242                 index = sample_action_pos;
5243                 if (index != 0)
5244                         memcpy(actions_pre, actions,
5245                                sizeof(struct rte_flow_action) * index);
5246         }
5247         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5248          * For CX6DX and above, metadata registers Cx preserve their value,
5249          * add an extra tag action for NIC-RX and E-Switch Domain.
5250          */
5251         if (add_tag) {
5252                 /* Prepare the prefix tag action. */
5253                 append_index++;
5254                 set_tag = (void *)(actions_pre + actions_n + append_index);
5255                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5256                 if (ret < 0)
5257                         return ret;
5258                 mlx5_ipool_malloc(priv->sh->ipool
5259                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5260                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5261                         .id = ret,
5262                         .data = tag_id,
5263                 };
5264                 /* Prepare the suffix subflow items. */
5265                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5266                 tag_spec->data = tag_id;
5267                 tag_spec->id = set_tag->id;
5268                 tag_mask = tag_spec + 1;
5269                 tag_mask->data = UINT32_MAX;
5270                 sfx_items[0] = (struct rte_flow_item){
5271                         .type = (enum rte_flow_item_type)
5272                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5273                         .spec = tag_spec,
5274                         .last = NULL,
5275                         .mask = tag_mask,
5276                 };
5277                 sfx_items[1] = (struct rte_flow_item){
5278                         .type = (enum rte_flow_item_type)
5279                                 RTE_FLOW_ITEM_TYPE_END,
5280                 };
5281                 /* Prepare the tag action in prefix subflow. */
5282                 actions_pre[index++] =
5283                         (struct rte_flow_action){
5284                         .type = (enum rte_flow_action_type)
5285                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5286                         .conf = set_tag,
5287                 };
5288         }
5289         memcpy(actions_pre + index, actions + sample_action_pos,
5290                sizeof(struct rte_flow_action));
5291         index += 1;
5292         /* For the modify action after the sample action in E-Switch mirroring,
5293          * Add the extra jump action in prefix subflow and jump into the next
5294          * table, then do the modify action in the new table.
5295          */
5296         if (jump_table) {
5297                 /* Prepare the prefix jump action. */
5298                 append_index++;
5299                 jump_action = (void *)(actions_pre + actions_n + append_index);
5300                 jump_action->group = jump_table;
5301                 actions_pre[index++] =
5302                         (struct rte_flow_action){
5303                         .type = (enum rte_flow_action_type)
5304                                 RTE_FLOW_ACTION_TYPE_JUMP,
5305                         .conf = jump_action,
5306                 };
5307         }
5308         actions_pre[index] = (struct rte_flow_action){
5309                 .type = (enum rte_flow_action_type)
5310                         RTE_FLOW_ACTION_TYPE_END,
5311         };
5312         /* Put the actions after sample into Suffix flow. */
5313         memcpy(actions_sfx, actions + sample_action_pos + 1,
5314                sizeof(struct rte_flow_action) *
5315                (actions_n - sample_action_pos - 1));
5316         return tag_id;
5317 }
5318
5319 /**
5320  * The splitting for metadata feature.
5321  *
5322  * - Q/RSS action on NIC Rx should be split in order to pass by
5323  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5324  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5325  *
5326  * - All the actions on NIC Tx should have a mreg copy action to
5327  *   copy reg_a from WQE to reg_c[0].
5328  *
5329  * @param dev
5330  *   Pointer to Ethernet device.
5331  * @param[in] flow
5332  *   Parent flow structure pointer.
5333  * @param[in] attr
5334  *   Flow rule attributes.
5335  * @param[in] items
5336  *   Pattern specification (list terminated by the END pattern item).
5337  * @param[in] actions
5338  *   Associated actions (list terminated by the END action).
5339  * @param[in] flow_split_info
5340  *   Pointer to flow split info structure.
5341  * @param[out] error
5342  *   Perform verbose error reporting if not NULL.
5343  * @return
5344  *   0 on success, negative value otherwise
5345  */
5346 static int
5347 flow_create_split_metadata(struct rte_eth_dev *dev,
5348                            struct rte_flow *flow,
5349                            const struct rte_flow_attr *attr,
5350                            const struct rte_flow_item items[],
5351                            const struct rte_flow_action actions[],
5352                            struct mlx5_flow_split_info *flow_split_info,
5353                            struct rte_flow_error *error)
5354 {
5355         struct mlx5_priv *priv = dev->data->dev_private;
5356         struct mlx5_dev_config *config = &priv->config;
5357         const struct rte_flow_action *qrss = NULL;
5358         struct rte_flow_action *ext_actions = NULL;
5359         struct mlx5_flow *dev_flow = NULL;
5360         uint32_t qrss_id = 0;
5361         int mtr_sfx = 0;
5362         size_t act_size;
5363         int actions_n;
5364         int encap_idx;
5365         int ret;
5366
5367         /* Check whether extensive metadata feature is engaged. */
5368         if (!config->dv_flow_en ||
5369             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5370             !mlx5_flow_ext_mreg_supported(dev))
5371                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5372                                                actions, flow_split_info, error);
5373         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5374                                                            &encap_idx);
5375         if (qrss) {
5376                 /* Exclude hairpin flows from splitting. */
5377                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5378                         const struct rte_flow_action_queue *queue;
5379
5380                         queue = qrss->conf;
5381                         if (mlx5_rxq_get_type(dev, queue->index) ==
5382                             MLX5_RXQ_TYPE_HAIRPIN)
5383                                 qrss = NULL;
5384                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5385                         const struct rte_flow_action_rss *rss;
5386
5387                         rss = qrss->conf;
5388                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5389                             MLX5_RXQ_TYPE_HAIRPIN)
5390                                 qrss = NULL;
5391                 }
5392         }
5393         if (qrss) {
5394                 /* Check if it is in meter suffix table. */
5395                 mtr_sfx = attr->group == (attr->transfer ?
5396                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5397                           MLX5_FLOW_TABLE_LEVEL_METER);
5398                 /*
5399                  * Q/RSS action on NIC Rx should be split in order to pass by
5400                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5401                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5402                  */
5403                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5404                            sizeof(struct rte_flow_action_set_tag) +
5405                            sizeof(struct rte_flow_action_jump);
5406                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5407                                           SOCKET_ID_ANY);
5408                 if (!ext_actions)
5409                         return rte_flow_error_set(error, ENOMEM,
5410                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5411                                                   NULL, "no memory to split "
5412                                                   "metadata flow");
5413                 /*
5414                  * If we are the suffix flow of meter, tag already exist.
5415                  * Set the tag action to void.
5416                  */
5417                 if (mtr_sfx)
5418                         ext_actions[qrss - actions].type =
5419                                                 RTE_FLOW_ACTION_TYPE_VOID;
5420                 else
5421                         ext_actions[qrss - actions].type =
5422                                                 (enum rte_flow_action_type)
5423                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5424                 /*
5425                  * Create the new actions list with removed Q/RSS action
5426                  * and appended set tag and jump to register copy table
5427                  * (RX_CP_TBL). We should preallocate unique tag ID here
5428                  * in advance, because it is needed for set tag action.
5429                  */
5430                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5431                                                     qrss, actions_n, error);
5432                 if (!mtr_sfx && !qrss_id) {
5433                         ret = -rte_errno;
5434                         goto exit;
5435                 }
5436         } else if (attr->egress && !attr->transfer) {
5437                 /*
5438                  * All the actions on NIC Tx should have a metadata register
5439                  * copy action to copy reg_a from WQE to reg_c[meta]
5440                  */
5441                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5442                            sizeof(struct mlx5_flow_action_copy_mreg);
5443                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5444                                           SOCKET_ID_ANY);
5445                 if (!ext_actions)
5446                         return rte_flow_error_set(error, ENOMEM,
5447                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5448                                                   NULL, "no memory to split "
5449                                                   "metadata flow");
5450                 /* Create the action list appended with copy register. */
5451                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5452                                              actions_n, error, encap_idx);
5453                 if (ret < 0)
5454                         goto exit;
5455         }
5456         /* Add the unmodified original or prefix subflow. */
5457         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5458                                       items, ext_actions ? ext_actions :
5459                                       actions, flow_split_info, error);
5460         if (ret < 0)
5461                 goto exit;
5462         MLX5_ASSERT(dev_flow);
5463         if (qrss) {
5464                 const struct rte_flow_attr q_attr = {
5465                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5466                         .ingress = 1,
5467                 };
5468                 /* Internal PMD action to set register. */
5469                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5470                         .data = qrss_id,
5471                         .id = REG_NON,
5472                 };
5473                 struct rte_flow_item q_items[] = {
5474                         {
5475                                 .type = (enum rte_flow_item_type)
5476                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5477                                 .spec = &q_tag_spec,
5478                                 .last = NULL,
5479                                 .mask = NULL,
5480                         },
5481                         {
5482                                 .type = RTE_FLOW_ITEM_TYPE_END,
5483                         },
5484                 };
5485                 struct rte_flow_action q_actions[] = {
5486                         {
5487                                 .type = qrss->type,
5488                                 .conf = qrss->conf,
5489                         },
5490                         {
5491                                 .type = RTE_FLOW_ACTION_TYPE_END,
5492                         },
5493                 };
5494                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5495
5496                 /*
5497                  * Configure the tag item only if there is no meter subflow.
5498                  * Since tag is already marked in the meter suffix subflow
5499                  * we can just use the meter suffix items as is.
5500                  */
5501                 if (qrss_id) {
5502                         /* Not meter subflow. */
5503                         MLX5_ASSERT(!mtr_sfx);
5504                         /*
5505                          * Put unique id in prefix flow due to it is destroyed
5506                          * after suffix flow and id will be freed after there
5507                          * is no actual flows with this id and identifier
5508                          * reallocation becomes possible (for example, for
5509                          * other flows in other threads).
5510                          */
5511                         dev_flow->handle->split_flow_id = qrss_id;
5512                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5513                                                    error);
5514                         if (ret < 0)
5515                                 goto exit;
5516                         q_tag_spec.id = ret;
5517                 }
5518                 dev_flow = NULL;
5519                 /* Add suffix subflow to execute Q/RSS. */
5520                 flow_split_info->prefix_layers = layers;
5521                 flow_split_info->prefix_mark = 0;
5522                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5523                                               &q_attr, mtr_sfx ? items :
5524                                               q_items, q_actions,
5525                                               flow_split_info, error);
5526                 if (ret < 0)
5527                         goto exit;
5528                 /* qrss ID should be freed if failed. */
5529                 qrss_id = 0;
5530                 MLX5_ASSERT(dev_flow);
5531         }
5532
5533 exit:
5534         /*
5535          * We do not destroy the partially created sub_flows in case of error.
5536          * These ones are included into parent flow list and will be destroyed
5537          * by flow_drv_destroy.
5538          */
5539         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5540                         qrss_id);
5541         mlx5_free(ext_actions);
5542         return ret;
5543 }
5544
5545 /**
5546  * Create meter internal drop flow with the original pattern.
5547  *
5548  * @param dev
5549  *   Pointer to Ethernet device.
5550  * @param[in] flow
5551  *   Parent flow structure pointer.
5552  * @param[in] attr
5553  *   Flow rule attributes.
5554  * @param[in] items
5555  *   Pattern specification (list terminated by the END pattern item).
5556  * @param[in] flow_split_info
5557  *   Pointer to flow split info structure.
5558  * @param[in] fm
5559  *   Pointer to flow meter structure.
5560  * @param[out] error
5561  *   Perform verbose error reporting if not NULL.
5562  * @return
5563  *   0 on success, negative value otherwise
5564  */
5565 static uint32_t
5566 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5567                         struct rte_flow *flow,
5568                         const struct rte_flow_attr *attr,
5569                         const struct rte_flow_item items[],
5570                         struct mlx5_flow_split_info *flow_split_info,
5571                         struct mlx5_flow_meter_info *fm,
5572                         struct rte_flow_error *error)
5573 {
5574         struct mlx5_flow *dev_flow = NULL;
5575         struct rte_flow_attr drop_attr = *attr;
5576         struct rte_flow_action drop_actions[3];
5577         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5578
5579         MLX5_ASSERT(fm->drop_cnt);
5580         drop_actions[0].type =
5581                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5582         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5583         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5584         drop_actions[1].conf = NULL;
5585         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5586         drop_actions[2].conf = NULL;
5587         drop_split_info.external = false;
5588         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5589         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5590         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5591         return flow_create_split_inner(dev, flow, &dev_flow,
5592                                 &drop_attr, items, drop_actions,
5593                                 &drop_split_info, error);
5594 }
5595
5596 /**
5597  * The splitting for meter feature.
5598  *
5599  * - The meter flow will be split to two flows as prefix and
5600  *   suffix flow. The packets make sense only it pass the prefix
5601  *   meter action.
5602  *
5603  * - Reg_C_5 is used for the packet to match betweend prefix and
5604  *   suffix flow.
5605  *
5606  * @param dev
5607  *   Pointer to Ethernet device.
5608  * @param[in] flow
5609  *   Parent flow structure pointer.
5610  * @param[in] attr
5611  *   Flow rule attributes.
5612  * @param[in] items
5613  *   Pattern specification (list terminated by the END pattern item).
5614  * @param[in] actions
5615  *   Associated actions (list terminated by the END action).
5616  * @param[in] flow_split_info
5617  *   Pointer to flow split info structure.
5618  * @param[out] error
5619  *   Perform verbose error reporting if not NULL.
5620  * @return
5621  *   0 on success, negative value otherwise
5622  */
5623 static int
5624 flow_create_split_meter(struct rte_eth_dev *dev,
5625                         struct rte_flow *flow,
5626                         const struct rte_flow_attr *attr,
5627                         const struct rte_flow_item items[],
5628                         const struct rte_flow_action actions[],
5629                         struct mlx5_flow_split_info *flow_split_info,
5630                         struct rte_flow_error *error)
5631 {
5632         struct mlx5_priv *priv = dev->data->dev_private;
5633         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5634         struct rte_flow_action *sfx_actions = NULL;
5635         struct rte_flow_action *pre_actions = NULL;
5636         struct rte_flow_item *sfx_items = NULL;
5637         struct mlx5_flow *dev_flow = NULL;
5638         struct rte_flow_attr sfx_attr = *attr;
5639         struct mlx5_flow_meter_info *fm = NULL;
5640         uint8_t skip_scale_restore;
5641         bool has_mtr = false;
5642         bool has_modify = false;
5643         bool set_mtr_reg = true;
5644         uint32_t meter_id = 0;
5645         uint32_t mtr_idx = 0;
5646         uint32_t mtr_flow_id = 0;
5647         size_t act_size;
5648         size_t item_size;
5649         int actions_n = 0;
5650         int ret = 0;
5651
5652         if (priv->mtr_en)
5653                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5654                                                     &has_modify, &meter_id);
5655         if (has_mtr) {
5656                 if (flow->meter) {
5657                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5658                         if (!fm)
5659                                 return rte_flow_error_set(error, EINVAL,
5660                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5661                                                 NULL, "Meter not found.");
5662                 } else {
5663                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5664                         if (!fm)
5665                                 return rte_flow_error_set(error, EINVAL,
5666                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5667                                                 NULL, "Meter not found.");
5668                         ret = mlx5_flow_meter_attach(priv, fm,
5669                                                      &sfx_attr, error);
5670                         if (ret)
5671                                 return -rte_errno;
5672                         flow->meter = mtr_idx;
5673                 }
5674                 MLX5_ASSERT(wks);
5675                 wks->fm = fm;
5676                 /*
5677                  * If it isn't default-policy Meter, and
5678                  * 1. There's no action in flow to change
5679                  *    packet (modify/encap/decap etc.), OR
5680                  * 2. No drop count needed for this meter.
5681                  * no need to use regC to save meter id anymore.
5682                  */
5683                 if (!fm->def_policy && (!has_modify || !fm->drop_cnt))
5684                         set_mtr_reg = false;
5685                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5686                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5687                            sizeof(struct mlx5_rte_flow_action_set_tag);
5688                 /* Suffix items: tag, vlan, port id, end. */
5689 #define METER_SUFFIX_ITEM 4
5690                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5691                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5692                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5693                                           0, SOCKET_ID_ANY);
5694                 if (!sfx_actions)
5695                         return rte_flow_error_set(error, ENOMEM,
5696                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5697                                                   NULL, "no memory to split "
5698                                                   "meter flow");
5699                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5700                              act_size);
5701                 /* There's no suffix flow for meter of non-default policy. */
5702                 if (!fm->def_policy)
5703                         pre_actions = sfx_actions + 1;
5704                 else
5705                         pre_actions = sfx_actions + actions_n;
5706                 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr,
5707                                             items, sfx_items, actions,
5708                                             sfx_actions, pre_actions,
5709                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5710                                             error);
5711                 if (ret) {
5712                         ret = -rte_errno;
5713                         goto exit;
5714                 }
5715                 /* Add the prefix subflow. */
5716                 flow_split_info->prefix_mark = 0;
5717                 skip_scale_restore = flow_split_info->skip_scale;
5718                 flow_split_info->skip_scale |=
5719                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5720                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5721                                               attr, items, pre_actions,
5722                                               flow_split_info, error);
5723                 flow_split_info->skip_scale = skip_scale_restore;
5724                 if (ret) {
5725                         if (mtr_flow_id)
5726                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5727                         ret = -rte_errno;
5728                         goto exit;
5729                 }
5730                 if (mtr_flow_id) {
5731                         dev_flow->handle->split_flow_id = mtr_flow_id;
5732                         dev_flow->handle->is_meter_flow_id = 1;
5733                 }
5734                 if (!fm->def_policy) {
5735                         if (!set_mtr_reg && fm->drop_cnt)
5736                                 ret =
5737                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5738                                                         &sfx_attr, items,
5739                                                         flow_split_info,
5740                                                         fm, error);
5741                         goto exit;
5742                 }
5743                 /* Setting the sfx group atrr. */
5744                 sfx_attr.group = sfx_attr.transfer ?
5745                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5746                                  MLX5_FLOW_TABLE_LEVEL_METER;
5747                 flow_split_info->prefix_layers =
5748                                 flow_get_prefix_layer_flags(dev_flow);
5749                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5750                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5751         }
5752         /* Add the prefix subflow. */
5753         ret = flow_create_split_metadata(dev, flow,
5754                                          &sfx_attr, sfx_items ?
5755                                          sfx_items : items,
5756                                          sfx_actions ? sfx_actions : actions,
5757                                          flow_split_info, error);
5758 exit:
5759         if (sfx_actions)
5760                 mlx5_free(sfx_actions);
5761         return ret;
5762 }
5763
5764 /**
5765  * The splitting for sample feature.
5766  *
5767  * Once Sample action is detected in the action list, the flow actions should
5768  * be split into prefix sub flow and suffix sub flow.
5769  *
5770  * The original items remain in the prefix sub flow, all actions preceding the
5771  * sample action and the sample action itself will be copied to the prefix
5772  * sub flow, the actions following the sample action will be copied to the
5773  * suffix sub flow, Queue action always be located in the suffix sub flow.
5774  *
5775  * In order to make the packet from prefix sub flow matches with suffix sub
5776  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5777  * flow uses tag item with the unique flow id.
5778  *
5779  * @param dev
5780  *   Pointer to Ethernet device.
5781  * @param[in] flow
5782  *   Parent flow structure pointer.
5783  * @param[in] attr
5784  *   Flow rule attributes.
5785  * @param[in] items
5786  *   Pattern specification (list terminated by the END pattern item).
5787  * @param[in] actions
5788  *   Associated actions (list terminated by the END action).
5789  * @param[in] flow_split_info
5790  *   Pointer to flow split info structure.
5791  * @param[out] error
5792  *   Perform verbose error reporting if not NULL.
5793  * @return
5794  *   0 on success, negative value otherwise
5795  */
5796 static int
5797 flow_create_split_sample(struct rte_eth_dev *dev,
5798                          struct rte_flow *flow,
5799                          const struct rte_flow_attr *attr,
5800                          const struct rte_flow_item items[],
5801                          const struct rte_flow_action actions[],
5802                          struct mlx5_flow_split_info *flow_split_info,
5803                          struct rte_flow_error *error)
5804 {
5805         struct mlx5_priv *priv = dev->data->dev_private;
5806         struct rte_flow_action *sfx_actions = NULL;
5807         struct rte_flow_action *pre_actions = NULL;
5808         struct rte_flow_item *sfx_items = NULL;
5809         struct mlx5_flow *dev_flow = NULL;
5810         struct rte_flow_attr sfx_attr = *attr;
5811 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5812         struct mlx5_flow_dv_sample_resource *sample_res;
5813         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5814         struct mlx5_flow_tbl_resource *sfx_tbl;
5815 #endif
5816         size_t act_size;
5817         size_t item_size;
5818         uint32_t fdb_tx = 0;
5819         int32_t tag_id = 0;
5820         int actions_n = 0;
5821         int sample_action_pos;
5822         int qrss_action_pos;
5823         int add_tag = 0;
5824         int modify_after_mirror = 0;
5825         uint16_t jump_table = 0;
5826         const uint32_t next_ft_step = 1;
5827         int ret = 0;
5828
5829         if (priv->sampler_en)
5830                 actions_n = flow_check_match_action(actions, attr,
5831                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5832                                         &sample_action_pos, &qrss_action_pos,
5833                                         &modify_after_mirror);
5834         if (actions_n) {
5835                 /* The prefix actions must includes sample, tag, end. */
5836                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5837                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5838                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5839                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5840                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5841                                           item_size), 0, SOCKET_ID_ANY);
5842                 if (!sfx_actions)
5843                         return rte_flow_error_set(error, ENOMEM,
5844                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5845                                                   NULL, "no memory to split "
5846                                                   "sample flow");
5847                 /* The representor_id is -1 for uplink. */
5848                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5849                 /*
5850                  * When reg_c_preserve is set, metadata registers Cx preserve
5851                  * their value even through packet duplication.
5852                  */
5853                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
5854                 if (add_tag)
5855                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5856                                         + act_size);
5857                 if (modify_after_mirror)
5858                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
5859                                      next_ft_step;
5860                 pre_actions = sfx_actions + actions_n;
5861                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
5862                                                 actions, sfx_actions,
5863                                                 pre_actions, actions_n,
5864                                                 sample_action_pos,
5865                                                 qrss_action_pos, jump_table,
5866                                                 error);
5867                 if (tag_id < 0 || (add_tag && !tag_id)) {
5868                         ret = -rte_errno;
5869                         goto exit;
5870                 }
5871                 if (modify_after_mirror)
5872                         flow_split_info->skip_scale =
5873                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5874                 /* Add the prefix subflow. */
5875                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5876                                               items, pre_actions,
5877                                               flow_split_info, error);
5878                 if (ret) {
5879                         ret = -rte_errno;
5880                         goto exit;
5881                 }
5882                 dev_flow->handle->split_flow_id = tag_id;
5883 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5884                 if (!modify_after_mirror) {
5885                         /* Set the sfx group attr. */
5886                         sample_res = (struct mlx5_flow_dv_sample_resource *)
5887                                                 dev_flow->dv.sample_res;
5888                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
5889                                                 sample_res->normal_path_tbl;
5890                         sfx_tbl_data = container_of(sfx_tbl,
5891                                                 struct mlx5_flow_tbl_data_entry,
5892                                                 tbl);
5893                         sfx_attr.group = sfx_attr.transfer ?
5894                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
5895                 } else {
5896                         MLX5_ASSERT(attr->transfer);
5897                         sfx_attr.group = jump_table;
5898                 }
5899                 flow_split_info->prefix_layers =
5900                                 flow_get_prefix_layer_flags(dev_flow);
5901                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5902                 /* Suffix group level already be scaled with factor, set
5903                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
5904                  * again in translation.
5905                  */
5906                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5907 #endif
5908         }
5909         /* Add the suffix subflow. */
5910         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5911                                       sfx_items ? sfx_items : items,
5912                                       sfx_actions ? sfx_actions : actions,
5913                                       flow_split_info, error);
5914 exit:
5915         if (sfx_actions)
5916                 mlx5_free(sfx_actions);
5917         return ret;
5918 }
5919
5920 /**
5921  * Split the flow to subflow set. The splitters might be linked
5922  * in the chain, like this:
5923  * flow_create_split_outer() calls:
5924  *   flow_create_split_meter() calls:
5925  *     flow_create_split_metadata(meter_subflow_0) calls:
5926  *       flow_create_split_inner(metadata_subflow_0)
5927  *       flow_create_split_inner(metadata_subflow_1)
5928  *       flow_create_split_inner(metadata_subflow_2)
5929  *     flow_create_split_metadata(meter_subflow_1) calls:
5930  *       flow_create_split_inner(metadata_subflow_0)
5931  *       flow_create_split_inner(metadata_subflow_1)
5932  *       flow_create_split_inner(metadata_subflow_2)
5933  *
5934  * This provide flexible way to add new levels of flow splitting.
5935  * The all of successfully created subflows are included to the
5936  * parent flow dev_flow list.
5937  *
5938  * @param dev
5939  *   Pointer to Ethernet device.
5940  * @param[in] flow
5941  *   Parent flow structure pointer.
5942  * @param[in] attr
5943  *   Flow rule attributes.
5944  * @param[in] items
5945  *   Pattern specification (list terminated by the END pattern item).
5946  * @param[in] actions
5947  *   Associated actions (list terminated by the END action).
5948  * @param[in] flow_split_info
5949  *   Pointer to flow split info structure.
5950  * @param[out] error
5951  *   Perform verbose error reporting if not NULL.
5952  * @return
5953  *   0 on success, negative value otherwise
5954  */
5955 static int
5956 flow_create_split_outer(struct rte_eth_dev *dev,
5957                         struct rte_flow *flow,
5958                         const struct rte_flow_attr *attr,
5959                         const struct rte_flow_item items[],
5960                         const struct rte_flow_action actions[],
5961                         struct mlx5_flow_split_info *flow_split_info,
5962                         struct rte_flow_error *error)
5963 {
5964         int ret;
5965
5966         ret = flow_create_split_sample(dev, flow, attr, items,
5967                                        actions, flow_split_info, error);
5968         MLX5_ASSERT(ret <= 0);
5969         return ret;
5970 }
5971
5972 static inline struct mlx5_flow_tunnel *
5973 flow_tunnel_from_rule(const struct mlx5_flow *flow)
5974 {
5975         struct mlx5_flow_tunnel *tunnel;
5976
5977 #pragma GCC diagnostic push
5978 #pragma GCC diagnostic ignored "-Wcast-qual"
5979         tunnel = (typeof(tunnel))flow->tunnel;
5980 #pragma GCC diagnostic pop
5981
5982         return tunnel;
5983 }
5984
5985 /**
5986  * Adjust flow RSS workspace if needed.
5987  *
5988  * @param wks
5989  *   Pointer to thread flow work space.
5990  * @param rss_desc
5991  *   Pointer to RSS descriptor.
5992  * @param[in] nrssq_num
5993  *   New RSS queue number.
5994  *
5995  * @return
5996  *   0 on success, -1 otherwise and rte_errno is set.
5997  */
5998 static int
5999 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6000                           struct mlx5_flow_rss_desc *rss_desc,
6001                           uint32_t nrssq_num)
6002 {
6003         if (likely(nrssq_num <= wks->rssq_num))
6004                 return 0;
6005         rss_desc->queue = realloc(rss_desc->queue,
6006                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6007         if (!rss_desc->queue) {
6008                 rte_errno = ENOMEM;
6009                 return -1;
6010         }
6011         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6012         return 0;
6013 }
6014
6015 /**
6016  * Create a flow and add it to @p list.
6017  *
6018  * @param dev
6019  *   Pointer to Ethernet device.
6020  * @param list
6021  *   Pointer to a TAILQ flow list. If this parameter NULL,
6022  *   no list insertion occurred, flow is just created,
6023  *   this is caller's responsibility to track the
6024  *   created flow.
6025  * @param[in] attr
6026  *   Flow rule attributes.
6027  * @param[in] items
6028  *   Pattern specification (list terminated by the END pattern item).
6029  * @param[in] actions
6030  *   Associated actions (list terminated by the END action).
6031  * @param[in] external
6032  *   This flow rule is created by request external to PMD.
6033  * @param[out] error
6034  *   Perform verbose error reporting if not NULL.
6035  *
6036  * @return
6037  *   A flow index on success, 0 otherwise and rte_errno is set.
6038  */
6039 static uint32_t
6040 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
6041                  const struct rte_flow_attr *attr,
6042                  const struct rte_flow_item items[],
6043                  const struct rte_flow_action original_actions[],
6044                  bool external, struct rte_flow_error *error)
6045 {
6046         struct mlx5_priv *priv = dev->data->dev_private;
6047         struct rte_flow *flow = NULL;
6048         struct mlx5_flow *dev_flow;
6049         const struct rte_flow_action_rss *rss = NULL;
6050         struct mlx5_translated_action_handle
6051                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6052         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6053         union {
6054                 struct mlx5_flow_expand_rss buf;
6055                 uint8_t buffer[2048];
6056         } expand_buffer;
6057         union {
6058                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6059                 uint8_t buffer[2048];
6060         } actions_rx;
6061         union {
6062                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6063                 uint8_t buffer[2048];
6064         } actions_hairpin_tx;
6065         union {
6066                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6067                 uint8_t buffer[2048];
6068         } items_tx;
6069         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6070         struct mlx5_flow_rss_desc *rss_desc;
6071         const struct rte_flow_action *p_actions_rx;
6072         uint32_t i;
6073         uint32_t idx = 0;
6074         int hairpin_flow;
6075         struct rte_flow_attr attr_tx = { .priority = 0 };
6076         const struct rte_flow_action *actions;
6077         struct rte_flow_action *translated_actions = NULL;
6078         struct mlx5_flow_tunnel *tunnel;
6079         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6080         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6081         struct mlx5_flow_split_info flow_split_info = {
6082                 .external = !!external,
6083                 .skip_scale = 0,
6084                 .flow_idx = 0,
6085                 .prefix_mark = 0,
6086                 .prefix_layers = 0,
6087                 .table_id = 0
6088         };
6089         int ret;
6090
6091         MLX5_ASSERT(wks);
6092         rss_desc = &wks->rss_desc;
6093         ret = flow_action_handles_translate(dev, original_actions,
6094                                             indir_actions,
6095                                             &indir_actions_n,
6096                                             &translated_actions, error);
6097         if (ret < 0) {
6098                 MLX5_ASSERT(translated_actions == NULL);
6099                 return 0;
6100         }
6101         actions = translated_actions ? translated_actions : original_actions;
6102         p_actions_rx = actions;
6103         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6104         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6105                                 external, hairpin_flow, error);
6106         if (ret < 0)
6107                 goto error_before_hairpin_split;
6108         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
6109         if (!flow) {
6110                 rte_errno = ENOMEM;
6111                 goto error_before_hairpin_split;
6112         }
6113         if (hairpin_flow > 0) {
6114                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6115                         rte_errno = EINVAL;
6116                         goto error_before_hairpin_split;
6117                 }
6118                 flow_hairpin_split(dev, actions, actions_rx.actions,
6119                                    actions_hairpin_tx.actions, items_tx.items,
6120                                    idx);
6121                 p_actions_rx = actions_rx.actions;
6122         }
6123         flow_split_info.flow_idx = idx;
6124         flow->drv_type = flow_get_drv_type(dev, attr);
6125         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6126                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6127         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6128         /* RSS Action only works on NIC RX domain */
6129         if (attr->ingress && !attr->transfer)
6130                 rss = flow_get_rss_action(dev, p_actions_rx);
6131         if (rss) {
6132                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6133                         return 0;
6134                 /*
6135                  * The following information is required by
6136                  * mlx5_flow_hashfields_adjust() in advance.
6137                  */
6138                 rss_desc->level = rss->level;
6139                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6140                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6141         }
6142         flow->dev_handles = 0;
6143         if (rss && rss->types) {
6144                 unsigned int graph_root;
6145
6146                 graph_root = find_graph_root(items, rss->level);
6147                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6148                                            items, rss->types,
6149                                            mlx5_support_expansion, graph_root);
6150                 MLX5_ASSERT(ret > 0 &&
6151                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6152         } else {
6153                 buf->entries = 1;
6154                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6155         }
6156         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6157                                                       indir_actions_n);
6158         for (i = 0; i < buf->entries; ++i) {
6159                 /* Initialize flow split data. */
6160                 flow_split_info.prefix_layers = 0;
6161                 flow_split_info.prefix_mark = 0;
6162                 flow_split_info.skip_scale = 0;
6163                 /*
6164                  * The splitter may create multiple dev_flows,
6165                  * depending on configuration. In the simplest
6166                  * case it just creates unmodified original flow.
6167                  */
6168                 ret = flow_create_split_outer(dev, flow, attr,
6169                                               buf->entry[i].pattern,
6170                                               p_actions_rx, &flow_split_info,
6171                                               error);
6172                 if (ret < 0)
6173                         goto error;
6174                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6175                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6176                                                            p_actions_rx,
6177                                                            idx,
6178                                                            wks->flows[0].tunnel,
6179                                                            &default_miss_ctx,
6180                                                            error);
6181                         if (ret < 0) {
6182                                 mlx5_free(default_miss_ctx.queue);
6183                                 goto error;
6184                         }
6185                 }
6186         }
6187         /* Create the tx flow. */
6188         if (hairpin_flow) {
6189                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6190                 attr_tx.ingress = 0;
6191                 attr_tx.egress = 1;
6192                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6193                                          actions_hairpin_tx.actions,
6194                                          idx, error);
6195                 if (!dev_flow)
6196                         goto error;
6197                 dev_flow->flow = flow;
6198                 dev_flow->external = 0;
6199                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6200                               dev_flow->handle, next);
6201                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6202                                          items_tx.items,
6203                                          actions_hairpin_tx.actions, error);
6204                 if (ret < 0)
6205                         goto error;
6206         }
6207         /*
6208          * Update the metadata register copy table. If extensive
6209          * metadata feature is enabled and registers are supported
6210          * we might create the extra rte_flow for each unique
6211          * MARK/FLAG action ID.
6212          *
6213          * The table is updated for ingress Flows only, because
6214          * the egress Flows belong to the different device and
6215          * copy table should be updated in peer NIC Rx domain.
6216          */
6217         if (attr->ingress &&
6218             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6219                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6220                 if (ret)
6221                         goto error;
6222         }
6223         /*
6224          * If the flow is external (from application) OR device is started,
6225          * OR mreg discover, then apply immediately.
6226          */
6227         if (external || dev->data->dev_started ||
6228             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6229              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6230                 ret = flow_drv_apply(dev, flow, error);
6231                 if (ret < 0)
6232                         goto error;
6233         }
6234         if (list) {
6235                 rte_spinlock_lock(&priv->flow_list_lock);
6236                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
6237                              flow, next);
6238                 rte_spinlock_unlock(&priv->flow_list_lock);
6239         }
6240         flow_rxq_flags_set(dev, flow);
6241         rte_free(translated_actions);
6242         tunnel = flow_tunnel_from_rule(wks->flows);
6243         if (tunnel) {
6244                 flow->tunnel = 1;
6245                 flow->tunnel_id = tunnel->tunnel_id;
6246                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6247                 mlx5_free(default_miss_ctx.queue);
6248         }
6249         mlx5_flow_pop_thread_workspace();
6250         return idx;
6251 error:
6252         MLX5_ASSERT(flow);
6253         ret = rte_errno; /* Save rte_errno before cleanup. */
6254         flow_mreg_del_copy_action(dev, flow);
6255         flow_drv_destroy(dev, flow);
6256         if (rss_desc->shared_rss)
6257                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6258                         mlx5_ipool_get
6259                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6260                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6261         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
6262         rte_errno = ret; /* Restore rte_errno. */
6263         ret = rte_errno;
6264         rte_errno = ret;
6265         mlx5_flow_pop_thread_workspace();
6266 error_before_hairpin_split:
6267         rte_free(translated_actions);
6268         return 0;
6269 }
6270
6271 /**
6272  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6273  * incoming packets to table 1.
6274  *
6275  * Other flow rules, requested for group n, will be created in
6276  * e-switch table n+1.
6277  * Jump action to e-switch group n will be created to group n+1.
6278  *
6279  * Used when working in switchdev mode, to utilise advantages of table 1
6280  * and above.
6281  *
6282  * @param dev
6283  *   Pointer to Ethernet device.
6284  *
6285  * @return
6286  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6287  */
6288 struct rte_flow *
6289 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6290 {
6291         const struct rte_flow_attr attr = {
6292                 .group = 0,
6293                 .priority = 0,
6294                 .ingress = 1,
6295                 .egress = 0,
6296                 .transfer = 1,
6297         };
6298         const struct rte_flow_item pattern = {
6299                 .type = RTE_FLOW_ITEM_TYPE_END,
6300         };
6301         struct rte_flow_action_jump jump = {
6302                 .group = 1,
6303         };
6304         const struct rte_flow_action actions[] = {
6305                 {
6306                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6307                         .conf = &jump,
6308                 },
6309                 {
6310                         .type = RTE_FLOW_ACTION_TYPE_END,
6311                 },
6312         };
6313         struct mlx5_priv *priv = dev->data->dev_private;
6314         struct rte_flow_error error;
6315
6316         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
6317                                                    &attr, &pattern,
6318                                                    actions, false, &error);
6319 }
6320
6321 /**
6322  * Validate a flow supported by the NIC.
6323  *
6324  * @see rte_flow_validate()
6325  * @see rte_flow_ops
6326  */
6327 int
6328 mlx5_flow_validate(struct rte_eth_dev *dev,
6329                    const struct rte_flow_attr *attr,
6330                    const struct rte_flow_item items[],
6331                    const struct rte_flow_action original_actions[],
6332                    struct rte_flow_error *error)
6333 {
6334         int hairpin_flow;
6335         struct mlx5_translated_action_handle
6336                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6337         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6338         const struct rte_flow_action *actions;
6339         struct rte_flow_action *translated_actions = NULL;
6340         int ret = flow_action_handles_translate(dev, original_actions,
6341                                                 indir_actions,
6342                                                 &indir_actions_n,
6343                                                 &translated_actions, error);
6344
6345         if (ret)
6346                 return ret;
6347         actions = translated_actions ? translated_actions : original_actions;
6348         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6349         ret = flow_drv_validate(dev, attr, items, actions,
6350                                 true, hairpin_flow, error);
6351         rte_free(translated_actions);
6352         return ret;
6353 }
6354
6355 /**
6356  * Create a flow.
6357  *
6358  * @see rte_flow_create()
6359  * @see rte_flow_ops
6360  */
6361 struct rte_flow *
6362 mlx5_flow_create(struct rte_eth_dev *dev,
6363                  const struct rte_flow_attr *attr,
6364                  const struct rte_flow_item items[],
6365                  const struct rte_flow_action actions[],
6366                  struct rte_flow_error *error)
6367 {
6368         struct mlx5_priv *priv = dev->data->dev_private;
6369
6370         /*
6371          * If the device is not started yet, it is not allowed to created a
6372          * flow from application. PMD default flows and traffic control flows
6373          * are not affected.
6374          */
6375         if (unlikely(!dev->data->dev_started)) {
6376                 DRV_LOG(DEBUG, "port %u is not started when "
6377                         "inserting a flow", dev->data->port_id);
6378                 rte_flow_error_set(error, ENODEV,
6379                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6380                                    NULL,
6381                                    "port not started");
6382                 return NULL;
6383         }
6384
6385         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
6386                                   attr, items, actions, true, error);
6387 }
6388
6389 /**
6390  * Destroy a flow in a list.
6391  *
6392  * @param dev
6393  *   Pointer to Ethernet device.
6394  * @param list
6395  *   Pointer to the Indexed flow list. If this parameter NULL,
6396  *   there is no flow removal from the list. Be noted that as
6397  *   flow is add to the indexed list, memory of the indexed
6398  *   list points to maybe changed as flow destroyed.
6399  * @param[in] flow_idx
6400  *   Index of flow to destroy.
6401  */
6402 static void
6403 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
6404                   uint32_t flow_idx)
6405 {
6406         struct mlx5_priv *priv = dev->data->dev_private;
6407         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6408                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
6409
6410         if (!flow)
6411                 return;
6412         /*
6413          * Update RX queue flags only if port is started, otherwise it is
6414          * already clean.
6415          */
6416         if (dev->data->dev_started)
6417                 flow_rxq_flags_trim(dev, flow);
6418         flow_drv_destroy(dev, flow);
6419         if (list) {
6420                 rte_spinlock_lock(&priv->flow_list_lock);
6421                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
6422                              flow_idx, flow, next);
6423                 rte_spinlock_unlock(&priv->flow_list_lock);
6424         }
6425         if (flow->tunnel) {
6426                 struct mlx5_flow_tunnel *tunnel;
6427
6428                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6429                 RTE_VERIFY(tunnel);
6430                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6431                         mlx5_flow_tunnel_free(dev, tunnel);
6432         }
6433         flow_mreg_del_copy_action(dev, flow);
6434         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6435 }
6436
6437 /**
6438  * Destroy all flows.
6439  *
6440  * @param dev
6441  *   Pointer to Ethernet device.
6442  * @param list
6443  *   Pointer to the Indexed flow list.
6444  * @param active
6445  *   If flushing is called avtively.
6446  */
6447 void
6448 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
6449 {
6450         uint32_t num_flushed = 0;
6451
6452         while (*list) {
6453                 flow_list_destroy(dev, list, *list);
6454                 num_flushed++;
6455         }
6456         if (active) {
6457                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6458                         dev->data->port_id, num_flushed);
6459         }
6460 }
6461
6462 /**
6463  * Stop all default actions for flows.
6464  *
6465  * @param dev
6466  *   Pointer to Ethernet device.
6467  */
6468 void
6469 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6470 {
6471         flow_mreg_del_default_copy_action(dev);
6472         flow_rxq_flags_clear(dev);
6473 }
6474
6475 /**
6476  * Start all default actions for flows.
6477  *
6478  * @param dev
6479  *   Pointer to Ethernet device.
6480  * @return
6481  *   0 on success, a negative errno value otherwise and rte_errno is set.
6482  */
6483 int
6484 mlx5_flow_start_default(struct rte_eth_dev *dev)
6485 {
6486         struct rte_flow_error error;
6487
6488         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6489         return flow_mreg_add_default_copy_action(dev, &error);
6490 }
6491
6492 /**
6493  * Release key of thread specific flow workspace data.
6494  */
6495 void
6496 flow_release_workspace(void *data)
6497 {
6498         struct mlx5_flow_workspace *wks = data;
6499         struct mlx5_flow_workspace *next;
6500
6501         while (wks) {
6502                 next = wks->next;
6503                 free(wks->rss_desc.queue);
6504                 free(wks);
6505                 wks = next;
6506         }
6507 }
6508
6509 /**
6510  * Get thread specific current flow workspace.
6511  *
6512  * @return pointer to thread specific flow workspace data, NULL on error.
6513  */
6514 struct mlx5_flow_workspace*
6515 mlx5_flow_get_thread_workspace(void)
6516 {
6517         struct mlx5_flow_workspace *data;
6518
6519         data = mlx5_flow_os_get_specific_workspace();
6520         MLX5_ASSERT(data && data->inuse);
6521         if (!data || !data->inuse)
6522                 DRV_LOG(ERR, "flow workspace not initialized.");
6523         return data;
6524 }
6525
6526 /**
6527  * Allocate and init new flow workspace.
6528  *
6529  * @return pointer to flow workspace data, NULL on error.
6530  */
6531 static struct mlx5_flow_workspace*
6532 flow_alloc_thread_workspace(void)
6533 {
6534         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6535
6536         if (!data) {
6537                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6538                         "memory.");
6539                 return NULL;
6540         }
6541         data->rss_desc.queue = calloc(1,
6542                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6543         if (!data->rss_desc.queue)
6544                 goto err;
6545         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6546         return data;
6547 err:
6548         if (data->rss_desc.queue)
6549                 free(data->rss_desc.queue);
6550         free(data);
6551         return NULL;
6552 }
6553
6554 /**
6555  * Get new thread specific flow workspace.
6556  *
6557  * If current workspace inuse, create new one and set as current.
6558  *
6559  * @return pointer to thread specific flow workspace data, NULL on error.
6560  */
6561 static struct mlx5_flow_workspace*
6562 mlx5_flow_push_thread_workspace(void)
6563 {
6564         struct mlx5_flow_workspace *curr;
6565         struct mlx5_flow_workspace *data;
6566
6567         curr = mlx5_flow_os_get_specific_workspace();
6568         if (!curr) {
6569                 data = flow_alloc_thread_workspace();
6570                 if (!data)
6571                         return NULL;
6572         } else if (!curr->inuse) {
6573                 data = curr;
6574         } else if (curr->next) {
6575                 data = curr->next;
6576         } else {
6577                 data = flow_alloc_thread_workspace();
6578                 if (!data)
6579                         return NULL;
6580                 curr->next = data;
6581                 data->prev = curr;
6582         }
6583         data->inuse = 1;
6584         data->flow_idx = 0;
6585         /* Set as current workspace */
6586         if (mlx5_flow_os_set_specific_workspace(data))
6587                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6588         return data;
6589 }
6590
6591 /**
6592  * Close current thread specific flow workspace.
6593  *
6594  * If previous workspace available, set it as current.
6595  *
6596  * @return pointer to thread specific flow workspace data, NULL on error.
6597  */
6598 static void
6599 mlx5_flow_pop_thread_workspace(void)
6600 {
6601         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6602
6603         if (!data)
6604                 return;
6605         if (!data->inuse) {
6606                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6607                 return;
6608         }
6609         data->inuse = 0;
6610         if (!data->prev)
6611                 return;
6612         if (mlx5_flow_os_set_specific_workspace(data->prev))
6613                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6614 }
6615
6616 /**
6617  * Verify the flow list is empty
6618  *
6619  * @param dev
6620  *  Pointer to Ethernet device.
6621  *
6622  * @return the number of flows not released.
6623  */
6624 int
6625 mlx5_flow_verify(struct rte_eth_dev *dev)
6626 {
6627         struct mlx5_priv *priv = dev->data->dev_private;
6628         struct rte_flow *flow;
6629         uint32_t idx;
6630         int ret = 0;
6631
6632         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6633                       flow, next) {
6634                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6635                         dev->data->port_id, (void *)flow);
6636                 ++ret;
6637         }
6638         return ret;
6639 }
6640
6641 /**
6642  * Enable default hairpin egress flow.
6643  *
6644  * @param dev
6645  *   Pointer to Ethernet device.
6646  * @param queue
6647  *   The queue index.
6648  *
6649  * @return
6650  *   0 on success, a negative errno value otherwise and rte_errno is set.
6651  */
6652 int
6653 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6654                             uint32_t queue)
6655 {
6656         struct mlx5_priv *priv = dev->data->dev_private;
6657         const struct rte_flow_attr attr = {
6658                 .egress = 1,
6659                 .priority = 0,
6660         };
6661         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6662                 .queue = queue,
6663         };
6664         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6665                 .queue = UINT32_MAX,
6666         };
6667         struct rte_flow_item items[] = {
6668                 {
6669                         .type = (enum rte_flow_item_type)
6670                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6671                         .spec = &queue_spec,
6672                         .last = NULL,
6673                         .mask = &queue_mask,
6674                 },
6675                 {
6676                         .type = RTE_FLOW_ITEM_TYPE_END,
6677                 },
6678         };
6679         struct rte_flow_action_jump jump = {
6680                 .group = MLX5_HAIRPIN_TX_TABLE,
6681         };
6682         struct rte_flow_action actions[2];
6683         uint32_t flow_idx;
6684         struct rte_flow_error error;
6685
6686         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6687         actions[0].conf = &jump;
6688         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6689         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6690                                 &attr, items, actions, false, &error);
6691         if (!flow_idx) {
6692                 DRV_LOG(DEBUG,
6693                         "Failed to create ctrl flow: rte_errno(%d),"
6694                         " type(%d), message(%s)",
6695                         rte_errno, error.type,
6696                         error.message ? error.message : " (no stated reason)");
6697                 return -rte_errno;
6698         }
6699         return 0;
6700 }
6701
6702 /**
6703  * Enable a control flow configured from the control plane.
6704  *
6705  * @param dev
6706  *   Pointer to Ethernet device.
6707  * @param eth_spec
6708  *   An Ethernet flow spec to apply.
6709  * @param eth_mask
6710  *   An Ethernet flow mask to apply.
6711  * @param vlan_spec
6712  *   A VLAN flow spec to apply.
6713  * @param vlan_mask
6714  *   A VLAN flow mask to apply.
6715  *
6716  * @return
6717  *   0 on success, a negative errno value otherwise and rte_errno is set.
6718  */
6719 int
6720 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6721                     struct rte_flow_item_eth *eth_spec,
6722                     struct rte_flow_item_eth *eth_mask,
6723                     struct rte_flow_item_vlan *vlan_spec,
6724                     struct rte_flow_item_vlan *vlan_mask)
6725 {
6726         struct mlx5_priv *priv = dev->data->dev_private;
6727         const struct rte_flow_attr attr = {
6728                 .ingress = 1,
6729                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6730         };
6731         struct rte_flow_item items[] = {
6732                 {
6733                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6734                         .spec = eth_spec,
6735                         .last = NULL,
6736                         .mask = eth_mask,
6737                 },
6738                 {
6739                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6740                                               RTE_FLOW_ITEM_TYPE_END,
6741                         .spec = vlan_spec,
6742                         .last = NULL,
6743                         .mask = vlan_mask,
6744                 },
6745                 {
6746                         .type = RTE_FLOW_ITEM_TYPE_END,
6747                 },
6748         };
6749         uint16_t queue[priv->reta_idx_n];
6750         struct rte_flow_action_rss action_rss = {
6751                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6752                 .level = 0,
6753                 .types = priv->rss_conf.rss_hf,
6754                 .key_len = priv->rss_conf.rss_key_len,
6755                 .queue_num = priv->reta_idx_n,
6756                 .key = priv->rss_conf.rss_key,
6757                 .queue = queue,
6758         };
6759         struct rte_flow_action actions[] = {
6760                 {
6761                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6762                         .conf = &action_rss,
6763                 },
6764                 {
6765                         .type = RTE_FLOW_ACTION_TYPE_END,
6766                 },
6767         };
6768         uint32_t flow_idx;
6769         struct rte_flow_error error;
6770         unsigned int i;
6771
6772         if (!priv->reta_idx_n || !priv->rxqs_n) {
6773                 return 0;
6774         }
6775         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6776                 action_rss.types = 0;
6777         for (i = 0; i != priv->reta_idx_n; ++i)
6778                 queue[i] = (*priv->reta_idx)[i];
6779         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6780                                 &attr, items, actions, false, &error);
6781         if (!flow_idx)
6782                 return -rte_errno;
6783         return 0;
6784 }
6785
6786 /**
6787  * Enable a flow control configured from the control plane.
6788  *
6789  * @param dev
6790  *   Pointer to Ethernet device.
6791  * @param eth_spec
6792  *   An Ethernet flow spec to apply.
6793  * @param eth_mask
6794  *   An Ethernet flow mask to apply.
6795  *
6796  * @return
6797  *   0 on success, a negative errno value otherwise and rte_errno is set.
6798  */
6799 int
6800 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6801                struct rte_flow_item_eth *eth_spec,
6802                struct rte_flow_item_eth *eth_mask)
6803 {
6804         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6805 }
6806
6807 /**
6808  * Create default miss flow rule matching lacp traffic
6809  *
6810  * @param dev
6811  *   Pointer to Ethernet device.
6812  * @param eth_spec
6813  *   An Ethernet flow spec to apply.
6814  *
6815  * @return
6816  *   0 on success, a negative errno value otherwise and rte_errno is set.
6817  */
6818 int
6819 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6820 {
6821         struct mlx5_priv *priv = dev->data->dev_private;
6822         /*
6823          * The LACP matching is done by only using ether type since using
6824          * a multicast dst mac causes kernel to give low priority to this flow.
6825          */
6826         static const struct rte_flow_item_eth lacp_spec = {
6827                 .type = RTE_BE16(0x8809),
6828         };
6829         static const struct rte_flow_item_eth lacp_mask = {
6830                 .type = 0xffff,
6831         };
6832         const struct rte_flow_attr attr = {
6833                 .ingress = 1,
6834         };
6835         struct rte_flow_item items[] = {
6836                 {
6837                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6838                         .spec = &lacp_spec,
6839                         .mask = &lacp_mask,
6840                 },
6841                 {
6842                         .type = RTE_FLOW_ITEM_TYPE_END,
6843                 },
6844         };
6845         struct rte_flow_action actions[] = {
6846                 {
6847                         .type = (enum rte_flow_action_type)
6848                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6849                 },
6850                 {
6851                         .type = RTE_FLOW_ACTION_TYPE_END,
6852                 },
6853         };
6854         struct rte_flow_error error;
6855         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6856                                 &attr, items, actions, false, &error);
6857
6858         if (!flow_idx)
6859                 return -rte_errno;
6860         return 0;
6861 }
6862
6863 /**
6864  * Destroy a flow.
6865  *
6866  * @see rte_flow_destroy()
6867  * @see rte_flow_ops
6868  */
6869 int
6870 mlx5_flow_destroy(struct rte_eth_dev *dev,
6871                   struct rte_flow *flow,
6872                   struct rte_flow_error *error __rte_unused)
6873 {
6874         struct mlx5_priv *priv = dev->data->dev_private;
6875
6876         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6877         return 0;
6878 }
6879
6880 /**
6881  * Destroy all flows.
6882  *
6883  * @see rte_flow_flush()
6884  * @see rte_flow_ops
6885  */
6886 int
6887 mlx5_flow_flush(struct rte_eth_dev *dev,
6888                 struct rte_flow_error *error __rte_unused)
6889 {
6890         struct mlx5_priv *priv = dev->data->dev_private;
6891
6892         mlx5_flow_list_flush(dev, &priv->flows, false);
6893         return 0;
6894 }
6895
6896 /**
6897  * Isolated mode.
6898  *
6899  * @see rte_flow_isolate()
6900  * @see rte_flow_ops
6901  */
6902 int
6903 mlx5_flow_isolate(struct rte_eth_dev *dev,
6904                   int enable,
6905                   struct rte_flow_error *error)
6906 {
6907         struct mlx5_priv *priv = dev->data->dev_private;
6908
6909         if (dev->data->dev_started) {
6910                 rte_flow_error_set(error, EBUSY,
6911                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6912                                    NULL,
6913                                    "port must be stopped first");
6914                 return -rte_errno;
6915         }
6916         priv->isolated = !!enable;
6917         if (enable)
6918                 dev->dev_ops = &mlx5_dev_ops_isolate;
6919         else
6920                 dev->dev_ops = &mlx5_dev_ops;
6921
6922         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6923         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6924
6925         return 0;
6926 }
6927
6928 /**
6929  * Query a flow.
6930  *
6931  * @see rte_flow_query()
6932  * @see rte_flow_ops
6933  */
6934 static int
6935 flow_drv_query(struct rte_eth_dev *dev,
6936                uint32_t flow_idx,
6937                const struct rte_flow_action *actions,
6938                void *data,
6939                struct rte_flow_error *error)
6940 {
6941         struct mlx5_priv *priv = dev->data->dev_private;
6942         const struct mlx5_flow_driver_ops *fops;
6943         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6944                                                [MLX5_IPOOL_RTE_FLOW],
6945                                                flow_idx);
6946         enum mlx5_flow_drv_type ftype;
6947
6948         if (!flow) {
6949                 return rte_flow_error_set(error, ENOENT,
6950                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6951                           NULL,
6952                           "invalid flow handle");
6953         }
6954         ftype = flow->drv_type;
6955         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6956         fops = flow_get_drv_ops(ftype);
6957
6958         return fops->query(dev, flow, actions, data, error);
6959 }
6960
6961 /**
6962  * Query a flow.
6963  *
6964  * @see rte_flow_query()
6965  * @see rte_flow_ops
6966  */
6967 int
6968 mlx5_flow_query(struct rte_eth_dev *dev,
6969                 struct rte_flow *flow,
6970                 const struct rte_flow_action *actions,
6971                 void *data,
6972                 struct rte_flow_error *error)
6973 {
6974         int ret;
6975
6976         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6977                              error);
6978         if (ret < 0)
6979                 return ret;
6980         return 0;
6981 }
6982
6983 /**
6984  * Get rte_flow callbacks.
6985  *
6986  * @param dev
6987  *   Pointer to Ethernet device structure.
6988  * @param ops
6989  *   Pointer to operation-specific structure.
6990  *
6991  * @return 0
6992  */
6993 int
6994 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
6995                   const struct rte_flow_ops **ops)
6996 {
6997         *ops = &mlx5_flow_ops;
6998         return 0;
6999 }
7000
7001 /**
7002  * Validate meter policy actions.
7003  * Dispatcher for action type specific validation.
7004  *
7005  * @param[in] dev
7006  *   Pointer to the Ethernet device structure.
7007  * @param[in] action
7008  *   The meter policy action object to validate.
7009  * @param[in] attr
7010  *   Attributes of flow to determine steering domain.
7011  * @param[out] is_rss
7012  *   Is RSS or not.
7013  * @param[out] domain_bitmap
7014  *   Domain bitmap.
7015  * @param[out] is_def_policy
7016  *   Is default policy or not.
7017  * @param[out] error
7018  *   Perform verbose error reporting if not NULL. Initialized in case of
7019  *   error only.
7020  *
7021  * @return
7022  *   0 on success, otherwise negative errno value.
7023  */
7024 int
7025 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7026                         const struct rte_flow_action *actions[RTE_COLORS],
7027                         struct rte_flow_attr *attr,
7028                         bool *is_rss,
7029                         uint8_t *domain_bitmap,
7030                         bool *is_def_policy,
7031                         struct rte_mtr_error *error)
7032 {
7033         const struct mlx5_flow_driver_ops *fops;
7034
7035         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7036         return fops->validate_mtr_acts(dev, actions, attr,
7037                         is_rss, domain_bitmap, is_def_policy, error);
7038 }
7039
7040 /**
7041  * Destroy the meter table set.
7042  *
7043  * @param[in] dev
7044  *   Pointer to Ethernet device.
7045  * @param[in] mtr_policy
7046  *   Meter policy struct.
7047  */
7048 void
7049 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7050                       struct mlx5_flow_meter_policy *mtr_policy)
7051 {
7052         const struct mlx5_flow_driver_ops *fops;
7053
7054         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7055         fops->destroy_mtr_acts(dev, mtr_policy);
7056 }
7057
7058 /**
7059  * Create policy action, lock free,
7060  * (mutex should be acquired by caller).
7061  * Dispatcher for action type specific call.
7062  *
7063  * @param[in] dev
7064  *   Pointer to the Ethernet device structure.
7065  * @param[in] mtr_policy
7066  *   Meter policy struct.
7067  * @param[in] action
7068  *   Action specification used to create meter actions.
7069  * @param[out] error
7070  *   Perform verbose error reporting if not NULL. Initialized in case of
7071  *   error only.
7072  *
7073  * @return
7074  *   0 on success, otherwise negative errno value.
7075  */
7076 int
7077 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7078                       struct mlx5_flow_meter_policy *mtr_policy,
7079                       const struct rte_flow_action *actions[RTE_COLORS],
7080                       struct rte_mtr_error *error)
7081 {
7082         const struct mlx5_flow_driver_ops *fops;
7083
7084         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7085         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7086 }
7087
7088 /**
7089  * Create policy rules, lock free,
7090  * (mutex should be acquired by caller).
7091  * Dispatcher for action type specific call.
7092  *
7093  * @param[in] dev
7094  *   Pointer to the Ethernet device structure.
7095  * @param[in] mtr_policy
7096  *   Meter policy struct.
7097  *
7098  * @return
7099  *   0 on success, -1 otherwise.
7100  */
7101 int
7102 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7103                              struct mlx5_flow_meter_policy *mtr_policy)
7104 {
7105         const struct mlx5_flow_driver_ops *fops;
7106
7107         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7108         return fops->create_policy_rules(dev, mtr_policy);
7109 }
7110
7111 /**
7112  * Destroy policy rules, lock free,
7113  * (mutex should be acquired by caller).
7114  * Dispatcher for action type specific call.
7115  *
7116  * @param[in] dev
7117  *   Pointer to the Ethernet device structure.
7118  * @param[in] mtr_policy
7119  *   Meter policy struct.
7120  */
7121 void
7122 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7123                              struct mlx5_flow_meter_policy *mtr_policy)
7124 {
7125         const struct mlx5_flow_driver_ops *fops;
7126
7127         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7128         fops->destroy_policy_rules(dev, mtr_policy);
7129 }
7130
7131 /**
7132  * Destroy the default policy table set.
7133  *
7134  * @param[in] dev
7135  *   Pointer to Ethernet device.
7136  */
7137 void
7138 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7139 {
7140         const struct mlx5_flow_driver_ops *fops;
7141
7142         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7143         fops->destroy_def_policy(dev);
7144 }
7145
7146 /**
7147  * Destroy the default policy table set.
7148  *
7149  * @param[in] dev
7150  *   Pointer to Ethernet device.
7151  *
7152  * @return
7153  *   0 on success, -1 otherwise.
7154  */
7155 int
7156 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7157 {
7158         const struct mlx5_flow_driver_ops *fops;
7159
7160         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7161         return fops->create_def_policy(dev);
7162 }
7163
7164 /**
7165  * Create the needed meter and suffix tables.
7166  *
7167  * @param[in] dev
7168  *   Pointer to Ethernet device.
7169  *
7170  * @return
7171  *   0 on success, -1 otherwise.
7172  */
7173 int
7174 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7175                         struct mlx5_flow_meter_info *fm,
7176                         uint32_t mtr_idx,
7177                         uint8_t domain_bitmap)
7178 {
7179         const struct mlx5_flow_driver_ops *fops;
7180
7181         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7182         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7183 }
7184
7185 /**
7186  * Destroy the meter table set.
7187  *
7188  * @param[in] dev
7189  *   Pointer to Ethernet device.
7190  * @param[in] tbl
7191  *   Pointer to the meter table set.
7192  */
7193 void
7194 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7195                            struct mlx5_flow_meter_info *fm)
7196 {
7197         const struct mlx5_flow_driver_ops *fops;
7198
7199         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7200         fops->destroy_mtr_tbls(dev, fm);
7201 }
7202
7203 /**
7204  * Destroy the global meter drop table.
7205  *
7206  * @param[in] dev
7207  *   Pointer to Ethernet device.
7208  */
7209 void
7210 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7211 {
7212         const struct mlx5_flow_driver_ops *fops;
7213
7214         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7215         fops->destroy_mtr_drop_tbls(dev);
7216 }
7217
7218 /**
7219  * Allocate the needed aso flow meter id.
7220  *
7221  * @param[in] dev
7222  *   Pointer to Ethernet device.
7223  *
7224  * @return
7225  *   Index to aso flow meter on success, NULL otherwise.
7226  */
7227 uint32_t
7228 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7229 {
7230         const struct mlx5_flow_driver_ops *fops;
7231
7232         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7233         return fops->create_meter(dev);
7234 }
7235
7236 /**
7237  * Free the aso flow meter id.
7238  *
7239  * @param[in] dev
7240  *   Pointer to Ethernet device.
7241  * @param[in] mtr_idx
7242  *  Index to aso flow meter to be free.
7243  *
7244  * @return
7245  *   0 on success.
7246  */
7247 void
7248 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7249 {
7250         const struct mlx5_flow_driver_ops *fops;
7251
7252         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7253         fops->free_meter(dev, mtr_idx);
7254 }
7255
7256 /**
7257  * Allocate a counter.
7258  *
7259  * @param[in] dev
7260  *   Pointer to Ethernet device structure.
7261  *
7262  * @return
7263  *   Index to allocated counter  on success, 0 otherwise.
7264  */
7265 uint32_t
7266 mlx5_counter_alloc(struct rte_eth_dev *dev)
7267 {
7268         const struct mlx5_flow_driver_ops *fops;
7269         struct rte_flow_attr attr = { .transfer = 0 };
7270
7271         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7272                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7273                 return fops->counter_alloc(dev);
7274         }
7275         DRV_LOG(ERR,
7276                 "port %u counter allocate is not supported.",
7277                  dev->data->port_id);
7278         return 0;
7279 }
7280
7281 /**
7282  * Free a counter.
7283  *
7284  * @param[in] dev
7285  *   Pointer to Ethernet device structure.
7286  * @param[in] cnt
7287  *   Index to counter to be free.
7288  */
7289 void
7290 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7291 {
7292         const struct mlx5_flow_driver_ops *fops;
7293         struct rte_flow_attr attr = { .transfer = 0 };
7294
7295         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7296                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7297                 fops->counter_free(dev, cnt);
7298                 return;
7299         }
7300         DRV_LOG(ERR,
7301                 "port %u counter free is not supported.",
7302                  dev->data->port_id);
7303 }
7304
7305 /**
7306  * Query counter statistics.
7307  *
7308  * @param[in] dev
7309  *   Pointer to Ethernet device structure.
7310  * @param[in] cnt
7311  *   Index to counter to query.
7312  * @param[in] clear
7313  *   Set to clear counter statistics.
7314  * @param[out] pkts
7315  *   The counter hits packets number to save.
7316  * @param[out] bytes
7317  *   The counter hits bytes number to save.
7318  *
7319  * @return
7320  *   0 on success, a negative errno value otherwise.
7321  */
7322 int
7323 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7324                    bool clear, uint64_t *pkts, uint64_t *bytes)
7325 {
7326         const struct mlx5_flow_driver_ops *fops;
7327         struct rte_flow_attr attr = { .transfer = 0 };
7328
7329         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7330                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7331                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7332         }
7333         DRV_LOG(ERR,
7334                 "port %u counter query is not supported.",
7335                  dev->data->port_id);
7336         return -ENOTSUP;
7337 }
7338
7339 /**
7340  * Allocate a new memory for the counter values wrapped by all the needed
7341  * management.
7342  *
7343  * @param[in] sh
7344  *   Pointer to mlx5_dev_ctx_shared object.
7345  *
7346  * @return
7347  *   0 on success, a negative errno value otherwise.
7348  */
7349 static int
7350 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7351 {
7352         struct mlx5_devx_mkey_attr mkey_attr;
7353         struct mlx5_counter_stats_mem_mng *mem_mng;
7354         volatile struct flow_counter_stats *raw_data;
7355         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7356         int size = (sizeof(struct flow_counter_stats) *
7357                         MLX5_COUNTERS_PER_POOL +
7358                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7359                         sizeof(struct mlx5_counter_stats_mem_mng);
7360         size_t pgsize = rte_mem_page_size();
7361         uint8_t *mem;
7362         int i;
7363
7364         if (pgsize == (size_t)-1) {
7365                 DRV_LOG(ERR, "Failed to get mem page size");
7366                 rte_errno = ENOMEM;
7367                 return -ENOMEM;
7368         }
7369         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7370         if (!mem) {
7371                 rte_errno = ENOMEM;
7372                 return -ENOMEM;
7373         }
7374         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7375         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7376         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7377                                                  IBV_ACCESS_LOCAL_WRITE);
7378         if (!mem_mng->umem) {
7379                 rte_errno = errno;
7380                 mlx5_free(mem);
7381                 return -rte_errno;
7382         }
7383         memset(&mkey_attr, 0, sizeof(mkey_attr));
7384         mkey_attr.addr = (uintptr_t)mem;
7385         mkey_attr.size = size;
7386         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7387         mkey_attr.pd = sh->pdn;
7388         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7389         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7390         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7391         if (!mem_mng->dm) {
7392                 mlx5_os_umem_dereg(mem_mng->umem);
7393                 rte_errno = errno;
7394                 mlx5_free(mem);
7395                 return -rte_errno;
7396         }
7397         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7398         raw_data = (volatile struct flow_counter_stats *)mem;
7399         for (i = 0; i < raws_n; ++i) {
7400                 mem_mng->raws[i].mem_mng = mem_mng;
7401                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7402         }
7403         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7404                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7405                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7406                                  next);
7407         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7408         sh->cmng.mem_mng = mem_mng;
7409         return 0;
7410 }
7411
7412 /**
7413  * Set the statistic memory to the new counter pool.
7414  *
7415  * @param[in] sh
7416  *   Pointer to mlx5_dev_ctx_shared object.
7417  * @param[in] pool
7418  *   Pointer to the pool to set the statistic memory.
7419  *
7420  * @return
7421  *   0 on success, a negative errno value otherwise.
7422  */
7423 static int
7424 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7425                                struct mlx5_flow_counter_pool *pool)
7426 {
7427         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7428         /* Resize statistic memory once used out. */
7429         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7430             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7431                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7432                 return -1;
7433         }
7434         rte_spinlock_lock(&pool->sl);
7435         pool->raw = cmng->mem_mng->raws + pool->index %
7436                     MLX5_CNT_CONTAINER_RESIZE;
7437         rte_spinlock_unlock(&pool->sl);
7438         pool->raw_hw = NULL;
7439         return 0;
7440 }
7441
7442 #define MLX5_POOL_QUERY_FREQ_US 1000000
7443
7444 /**
7445  * Set the periodic procedure for triggering asynchronous batch queries for all
7446  * the counter pools.
7447  *
7448  * @param[in] sh
7449  *   Pointer to mlx5_dev_ctx_shared object.
7450  */
7451 void
7452 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7453 {
7454         uint32_t pools_n, us;
7455
7456         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7457         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7458         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7459         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7460                 sh->cmng.query_thread_on = 0;
7461                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7462         } else {
7463                 sh->cmng.query_thread_on = 1;
7464         }
7465 }
7466
7467 /**
7468  * The periodic procedure for triggering asynchronous batch queries for all the
7469  * counter pools. This function is probably called by the host thread.
7470  *
7471  * @param[in] arg
7472  *   The parameter for the alarm process.
7473  */
7474 void
7475 mlx5_flow_query_alarm(void *arg)
7476 {
7477         struct mlx5_dev_ctx_shared *sh = arg;
7478         int ret;
7479         uint16_t pool_index = sh->cmng.pool_index;
7480         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7481         struct mlx5_flow_counter_pool *pool;
7482         uint16_t n_valid;
7483
7484         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7485                 goto set_alarm;
7486         rte_spinlock_lock(&cmng->pool_update_sl);
7487         pool = cmng->pools[pool_index];
7488         n_valid = cmng->n_valid;
7489         rte_spinlock_unlock(&cmng->pool_update_sl);
7490         /* Set the statistic memory to the new created pool. */
7491         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7492                 goto set_alarm;
7493         if (pool->raw_hw)
7494                 /* There is a pool query in progress. */
7495                 goto set_alarm;
7496         pool->raw_hw =
7497                 LIST_FIRST(&sh->cmng.free_stat_raws);
7498         if (!pool->raw_hw)
7499                 /* No free counter statistics raw memory. */
7500                 goto set_alarm;
7501         /*
7502          * Identify the counters released between query trigger and query
7503          * handle more efficiently. The counter released in this gap period
7504          * should wait for a new round of query as the new arrived packets
7505          * will not be taken into account.
7506          */
7507         pool->query_gen++;
7508         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7509                                                MLX5_COUNTERS_PER_POOL,
7510                                                NULL, NULL,
7511                                                pool->raw_hw->mem_mng->dm->id,
7512                                                (void *)(uintptr_t)
7513                                                pool->raw_hw->data,
7514                                                sh->devx_comp,
7515                                                (uint64_t)(uintptr_t)pool);
7516         if (ret) {
7517                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7518                         " %d", pool->min_dcs->id);
7519                 pool->raw_hw = NULL;
7520                 goto set_alarm;
7521         }
7522         LIST_REMOVE(pool->raw_hw, next);
7523         sh->cmng.pending_queries++;
7524         pool_index++;
7525         if (pool_index >= n_valid)
7526                 pool_index = 0;
7527 set_alarm:
7528         sh->cmng.pool_index = pool_index;
7529         mlx5_set_query_alarm(sh);
7530 }
7531
7532 /**
7533  * Check and callback event for new aged flow in the counter pool
7534  *
7535  * @param[in] sh
7536  *   Pointer to mlx5_dev_ctx_shared object.
7537  * @param[in] pool
7538  *   Pointer to Current counter pool.
7539  */
7540 static void
7541 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7542                    struct mlx5_flow_counter_pool *pool)
7543 {
7544         struct mlx5_priv *priv;
7545         struct mlx5_flow_counter *cnt;
7546         struct mlx5_age_info *age_info;
7547         struct mlx5_age_param *age_param;
7548         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7549         struct mlx5_counter_stats_raw *prev = pool->raw;
7550         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7551         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7552         uint16_t expected = AGE_CANDIDATE;
7553         uint32_t i;
7554
7555         pool->time_of_last_age_check = curr_time;
7556         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7557                 cnt = MLX5_POOL_GET_CNT(pool, i);
7558                 age_param = MLX5_CNT_TO_AGE(cnt);
7559                 if (__atomic_load_n(&age_param->state,
7560                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7561                         continue;
7562                 if (cur->data[i].hits != prev->data[i].hits) {
7563                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7564                                          __ATOMIC_RELAXED);
7565                         continue;
7566                 }
7567                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7568                                        time_delta,
7569                                        __ATOMIC_RELAXED) <= age_param->timeout)
7570                         continue;
7571                 /**
7572                  * Hold the lock first, or if between the
7573                  * state AGE_TMOUT and tailq operation the
7574                  * release happened, the release procedure
7575                  * may delete a non-existent tailq node.
7576                  */
7577                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7578                 age_info = GET_PORT_AGE_INFO(priv);
7579                 rte_spinlock_lock(&age_info->aged_sl);
7580                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7581                                                 AGE_TMOUT, false,
7582                                                 __ATOMIC_RELAXED,
7583                                                 __ATOMIC_RELAXED)) {
7584                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7585                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7586                 }
7587                 rte_spinlock_unlock(&age_info->aged_sl);
7588         }
7589         mlx5_age_event_prepare(sh);
7590 }
7591
7592 /**
7593  * Handler for the HW respond about ready values from an asynchronous batch
7594  * query. This function is probably called by the host thread.
7595  *
7596  * @param[in] sh
7597  *   The pointer to the shared device context.
7598  * @param[in] async_id
7599  *   The Devx async ID.
7600  * @param[in] status
7601  *   The status of the completion.
7602  */
7603 void
7604 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7605                                   uint64_t async_id, int status)
7606 {
7607         struct mlx5_flow_counter_pool *pool =
7608                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7609         struct mlx5_counter_stats_raw *raw_to_free;
7610         uint8_t query_gen = pool->query_gen ^ 1;
7611         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7612         enum mlx5_counter_type cnt_type =
7613                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7614                                 MLX5_COUNTER_TYPE_ORIGIN;
7615
7616         if (unlikely(status)) {
7617                 raw_to_free = pool->raw_hw;
7618         } else {
7619                 raw_to_free = pool->raw;
7620                 if (pool->is_aged)
7621                         mlx5_flow_aging_check(sh, pool);
7622                 rte_spinlock_lock(&pool->sl);
7623                 pool->raw = pool->raw_hw;
7624                 rte_spinlock_unlock(&pool->sl);
7625                 /* Be sure the new raw counters data is updated in memory. */
7626                 rte_io_wmb();
7627                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7628                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7629                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7630                                      &pool->counters[query_gen], next);
7631                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7632                 }
7633         }
7634         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7635         pool->raw_hw = NULL;
7636         sh->cmng.pending_queries--;
7637 }
7638
7639 static int
7640 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7641                     const struct flow_grp_info *grp_info,
7642                     struct rte_flow_error *error)
7643 {
7644         if (grp_info->transfer && grp_info->external &&
7645             grp_info->fdb_def_rule) {
7646                 if (group == UINT32_MAX)
7647                         return rte_flow_error_set
7648                                                 (error, EINVAL,
7649                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7650                                                  NULL,
7651                                                  "group index not supported");
7652                 *table = group + 1;
7653         } else {
7654                 *table = group;
7655         }
7656         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7657         return 0;
7658 }
7659
7660 /**
7661  * Translate the rte_flow group index to HW table value.
7662  *
7663  * If tunnel offload is disabled, all group ids converted to flow table
7664  * id using the standard method.
7665  * If tunnel offload is enabled, group id can be converted using the
7666  * standard or tunnel conversion method. Group conversion method
7667  * selection depends on flags in `grp_info` parameter:
7668  * - Internal (grp_info.external == 0) groups conversion uses the
7669  *   standard method.
7670  * - Group ids in JUMP action converted with the tunnel conversion.
7671  * - Group id in rule attribute conversion depends on a rule type and
7672  *   group id value:
7673  *   ** non zero group attributes converted with the tunnel method
7674  *   ** zero group attribute in non-tunnel rule is converted using the
7675  *      standard method - there's only one root table
7676  *   ** zero group attribute in steer tunnel rule is converted with the
7677  *      standard method - single root table
7678  *   ** zero group attribute in match tunnel rule is a special OvS
7679  *      case: that value is used for portability reasons. That group
7680  *      id is converted with the tunnel conversion method.
7681  *
7682  * @param[in] dev
7683  *   Port device
7684  * @param[in] tunnel
7685  *   PMD tunnel offload object
7686  * @param[in] group
7687  *   rte_flow group index value.
7688  * @param[out] table
7689  *   HW table value.
7690  * @param[in] grp_info
7691  *   flags used for conversion
7692  * @param[out] error
7693  *   Pointer to error structure.
7694  *
7695  * @return
7696  *   0 on success, a negative errno value otherwise and rte_errno is set.
7697  */
7698 int
7699 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7700                          const struct mlx5_flow_tunnel *tunnel,
7701                          uint32_t group, uint32_t *table,
7702                          const struct flow_grp_info *grp_info,
7703                          struct rte_flow_error *error)
7704 {
7705         int ret;
7706         bool standard_translation;
7707
7708         if (!grp_info->skip_scale && grp_info->external &&
7709             group < MLX5_MAX_TABLES_EXTERNAL)
7710                 group *= MLX5_FLOW_TABLE_FACTOR;
7711         if (is_tunnel_offload_active(dev)) {
7712                 standard_translation = !grp_info->external ||
7713                                         grp_info->std_tbl_fix;
7714         } else {
7715                 standard_translation = true;
7716         }
7717         DRV_LOG(DEBUG,
7718                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7719                 dev->data->port_id, group, grp_info->transfer,
7720                 grp_info->external, grp_info->fdb_def_rule,
7721                 standard_translation ? "STANDARD" : "TUNNEL");
7722         if (standard_translation)
7723                 ret = flow_group_to_table(dev->data->port_id, group, table,
7724                                           grp_info, error);
7725         else
7726                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7727                                                       table, error);
7728
7729         return ret;
7730 }
7731
7732 /**
7733  * Discover availability of metadata reg_c's.
7734  *
7735  * Iteratively use test flows to check availability.
7736  *
7737  * @param[in] dev
7738  *   Pointer to the Ethernet device structure.
7739  *
7740  * @return
7741  *   0 on success, a negative errno value otherwise and rte_errno is set.
7742  */
7743 int
7744 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7745 {
7746         struct mlx5_priv *priv = dev->data->dev_private;
7747         struct mlx5_dev_config *config = &priv->config;
7748         enum modify_reg idx;
7749         int n = 0;
7750
7751         /* reg_c[0] and reg_c[1] are reserved. */
7752         config->flow_mreg_c[n++] = REG_C_0;
7753         config->flow_mreg_c[n++] = REG_C_1;
7754         /* Discover availability of other reg_c's. */
7755         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7756                 struct rte_flow_attr attr = {
7757                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7758                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7759                         .ingress = 1,
7760                 };
7761                 struct rte_flow_item items[] = {
7762                         [0] = {
7763                                 .type = RTE_FLOW_ITEM_TYPE_END,
7764                         },
7765                 };
7766                 struct rte_flow_action actions[] = {
7767                         [0] = {
7768                                 .type = (enum rte_flow_action_type)
7769                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7770                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7771                                         .src = REG_C_1,
7772                                         .dst = idx,
7773                                 },
7774                         },
7775                         [1] = {
7776                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7777                                 .conf = &(struct rte_flow_action_jump){
7778                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7779                                 },
7780                         },
7781                         [2] = {
7782                                 .type = RTE_FLOW_ACTION_TYPE_END,
7783                         },
7784                 };
7785                 uint32_t flow_idx;
7786                 struct rte_flow *flow;
7787                 struct rte_flow_error error;
7788
7789                 if (!config->dv_flow_en)
7790                         break;
7791                 /* Create internal flow, validation skips copy action. */
7792                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7793                                             actions, false, &error);
7794                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7795                                       flow_idx);
7796                 if (!flow)
7797                         continue;
7798                 config->flow_mreg_c[n++] = idx;
7799                 flow_list_destroy(dev, NULL, flow_idx);
7800         }
7801         for (; n < MLX5_MREG_C_NUM; ++n)
7802                 config->flow_mreg_c[n] = REG_NON;
7803         return 0;
7804 }
7805
7806 /**
7807  * Dump flow raw hw data to file
7808  *
7809  * @param[in] dev
7810  *    The pointer to Ethernet device.
7811  * @param[in] file
7812  *   A pointer to a file for output.
7813  * @param[out] error
7814  *   Perform verbose error reporting if not NULL. PMDs initialize this
7815  *   structure in case of error only.
7816  * @return
7817  *   0 on success, a nagative value otherwise.
7818  */
7819 int
7820 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
7821                    FILE *file,
7822                    struct rte_flow_error *error __rte_unused)
7823 {
7824         struct mlx5_priv *priv = dev->data->dev_private;
7825         struct mlx5_dev_ctx_shared *sh = priv->sh;
7826         uint32_t handle_idx;
7827         int ret;
7828         struct mlx5_flow_handle *dh;
7829         struct rte_flow *flow;
7830
7831         if (!priv->config.dv_flow_en) {
7832                 if (fputs("device dv flow disabled\n", file) <= 0)
7833                         return -errno;
7834                 return -ENOTSUP;
7835         }
7836
7837         /* dump all */
7838         if (!flow_idx)
7839                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
7840                                         sh->rx_domain,
7841                                         sh->tx_domain, file);
7842         /* dump one */
7843         flow = mlx5_ipool_get(priv->sh->ipool
7844                         [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
7845         if (!flow)
7846                 return -ENOENT;
7847
7848         handle_idx = flow->dev_handles;
7849         while (handle_idx) {
7850                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7851                                 handle_idx);
7852                 if (!dh)
7853                         return -ENOENT;
7854                 if (dh->drv_flow) {
7855                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
7856                                         file);
7857                         if (ret)
7858                                 return -ENOENT;
7859                 }
7860                 handle_idx = dh->next.next;
7861         }
7862         return 0;
7863 }
7864
7865 /**
7866  * Get aged-out flows.
7867  *
7868  * @param[in] dev
7869  *   Pointer to the Ethernet device structure.
7870  * @param[in] context
7871  *   The address of an array of pointers to the aged-out flows contexts.
7872  * @param[in] nb_countexts
7873  *   The length of context array pointers.
7874  * @param[out] error
7875  *   Perform verbose error reporting if not NULL. Initialized in case of
7876  *   error only.
7877  *
7878  * @return
7879  *   how many contexts get in success, otherwise negative errno value.
7880  *   if nb_contexts is 0, return the amount of all aged contexts.
7881  *   if nb_contexts is not 0 , return the amount of aged flows reported
7882  *   in the context array.
7883  */
7884 int
7885 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
7886                         uint32_t nb_contexts, struct rte_flow_error *error)
7887 {
7888         const struct mlx5_flow_driver_ops *fops;
7889         struct rte_flow_attr attr = { .transfer = 0 };
7890
7891         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7892                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7893                 return fops->get_aged_flows(dev, contexts, nb_contexts,
7894                                                     error);
7895         }
7896         DRV_LOG(ERR,
7897                 "port %u get aged flows is not supported.",
7898                  dev->data->port_id);
7899         return -ENOTSUP;
7900 }
7901
7902 /* Wrapper for driver action_validate op callback */
7903 static int
7904 flow_drv_action_validate(struct rte_eth_dev *dev,
7905                          const struct rte_flow_indir_action_conf *conf,
7906                          const struct rte_flow_action *action,
7907                          const struct mlx5_flow_driver_ops *fops,
7908                          struct rte_flow_error *error)
7909 {
7910         static const char err_msg[] = "indirect action validation unsupported";
7911
7912         if (!fops->action_validate) {
7913                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7914                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7915                                    NULL, err_msg);
7916                 return -rte_errno;
7917         }
7918         return fops->action_validate(dev, conf, action, error);
7919 }
7920
7921 /**
7922  * Destroys the shared action by handle.
7923  *
7924  * @param dev
7925  *   Pointer to Ethernet device structure.
7926  * @param[in] handle
7927  *   Handle for the indirect action object to be destroyed.
7928  * @param[out] error
7929  *   Perform verbose error reporting if not NULL. PMDs initialize this
7930  *   structure in case of error only.
7931  *
7932  * @return
7933  *   0 on success, a negative errno value otherwise and rte_errno is set.
7934  *
7935  * @note: wrapper for driver action_create op callback.
7936  */
7937 static int
7938 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
7939                            struct rte_flow_action_handle *handle,
7940                            struct rte_flow_error *error)
7941 {
7942         static const char err_msg[] = "indirect action destruction unsupported";
7943         struct rte_flow_attr attr = { .transfer = 0 };
7944         const struct mlx5_flow_driver_ops *fops =
7945                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7946
7947         if (!fops->action_destroy) {
7948                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7949                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7950                                    NULL, err_msg);
7951                 return -rte_errno;
7952         }
7953         return fops->action_destroy(dev, handle, error);
7954 }
7955
7956 /* Wrapper for driver action_destroy op callback */
7957 static int
7958 flow_drv_action_update(struct rte_eth_dev *dev,
7959                        struct rte_flow_action_handle *handle,
7960                        const void *update,
7961                        const struct mlx5_flow_driver_ops *fops,
7962                        struct rte_flow_error *error)
7963 {
7964         static const char err_msg[] = "indirect action update unsupported";
7965
7966         if (!fops->action_update) {
7967                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7968                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7969                                    NULL, err_msg);
7970                 return -rte_errno;
7971         }
7972         return fops->action_update(dev, handle, update, error);
7973 }
7974
7975 /* Wrapper for driver action_destroy op callback */
7976 static int
7977 flow_drv_action_query(struct rte_eth_dev *dev,
7978                       const struct rte_flow_action_handle *handle,
7979                       void *data,
7980                       const struct mlx5_flow_driver_ops *fops,
7981                       struct rte_flow_error *error)
7982 {
7983         static const char err_msg[] = "indirect action query unsupported";
7984
7985         if (!fops->action_query) {
7986                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7987                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7988                                    NULL, err_msg);
7989                 return -rte_errno;
7990         }
7991         return fops->action_query(dev, handle, data, error);
7992 }
7993
7994 /**
7995  * Create indirect action for reuse in multiple flow rules.
7996  *
7997  * @param dev
7998  *   Pointer to Ethernet device structure.
7999  * @param conf
8000  *   Pointer to indirect action object configuration.
8001  * @param[in] action
8002  *   Action configuration for indirect action object creation.
8003  * @param[out] error
8004  *   Perform verbose error reporting if not NULL. PMDs initialize this
8005  *   structure in case of error only.
8006  * @return
8007  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8008  */
8009 static struct rte_flow_action_handle *
8010 mlx5_action_handle_create(struct rte_eth_dev *dev,
8011                           const struct rte_flow_indir_action_conf *conf,
8012                           const struct rte_flow_action *action,
8013                           struct rte_flow_error *error)
8014 {
8015         static const char err_msg[] = "indirect action creation unsupported";
8016         struct rte_flow_attr attr = { .transfer = 0 };
8017         const struct mlx5_flow_driver_ops *fops =
8018                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8019
8020         if (flow_drv_action_validate(dev, conf, action, fops, error))
8021                 return NULL;
8022         if (!fops->action_create) {
8023                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8024                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8025                                    NULL, err_msg);
8026                 return NULL;
8027         }
8028         return fops->action_create(dev, conf, action, error);
8029 }
8030
8031 /**
8032  * Updates inplace the indirect action configuration pointed by *handle*
8033  * with the configuration provided as *update* argument.
8034  * The update of the indirect action configuration effects all flow rules
8035  * reusing the action via handle.
8036  *
8037  * @param dev
8038  *   Pointer to Ethernet device structure.
8039  * @param[in] handle
8040  *   Handle for the indirect action to be updated.
8041  * @param[in] update
8042  *   Action specification used to modify the action pointed by handle.
8043  *   *update* could be of same type with the action pointed by the *handle*
8044  *   handle argument, or some other structures like a wrapper, depending on
8045  *   the indirect action type.
8046  * @param[out] error
8047  *   Perform verbose error reporting if not NULL. PMDs initialize this
8048  *   structure in case of error only.
8049  *
8050  * @return
8051  *   0 on success, a negative errno value otherwise and rte_errno is set.
8052  */
8053 static int
8054 mlx5_action_handle_update(struct rte_eth_dev *dev,
8055                 struct rte_flow_action_handle *handle,
8056                 const void *update,
8057                 struct rte_flow_error *error)
8058 {
8059         struct rte_flow_attr attr = { .transfer = 0 };
8060         const struct mlx5_flow_driver_ops *fops =
8061                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8062         int ret;
8063
8064         ret = flow_drv_action_validate(dev, NULL,
8065                         (const struct rte_flow_action *)update, fops, error);
8066         if (ret)
8067                 return ret;
8068         return flow_drv_action_update(dev, handle, update, fops,
8069                                       error);
8070 }
8071
8072 /**
8073  * Query the indirect action by handle.
8074  *
8075  * This function allows retrieving action-specific data such as counters.
8076  * Data is gathered by special action which may be present/referenced in
8077  * more than one flow rule definition.
8078  *
8079  * see @RTE_FLOW_ACTION_TYPE_COUNT
8080  *
8081  * @param dev
8082  *   Pointer to Ethernet device structure.
8083  * @param[in] handle
8084  *   Handle for the indirect action to query.
8085  * @param[in, out] data
8086  *   Pointer to storage for the associated query data type.
8087  * @param[out] error
8088  *   Perform verbose error reporting if not NULL. PMDs initialize this
8089  *   structure in case of error only.
8090  *
8091  * @return
8092  *   0 on success, a negative errno value otherwise and rte_errno is set.
8093  */
8094 static int
8095 mlx5_action_handle_query(struct rte_eth_dev *dev,
8096                          const struct rte_flow_action_handle *handle,
8097                          void *data,
8098                          struct rte_flow_error *error)
8099 {
8100         struct rte_flow_attr attr = { .transfer = 0 };
8101         const struct mlx5_flow_driver_ops *fops =
8102                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8103
8104         return flow_drv_action_query(dev, handle, data, fops, error);
8105 }
8106
8107 /**
8108  * Destroy all indirect actions (shared RSS).
8109  *
8110  * @param dev
8111  *   Pointer to Ethernet device.
8112  *
8113  * @return
8114  *   0 on success, a negative errno value otherwise and rte_errno is set.
8115  */
8116 int
8117 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8118 {
8119         struct rte_flow_error error;
8120         struct mlx5_priv *priv = dev->data->dev_private;
8121         struct mlx5_shared_action_rss *shared_rss;
8122         int ret = 0;
8123         uint32_t idx;
8124
8125         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8126                       priv->rss_shared_actions, idx, shared_rss, next) {
8127                 ret |= mlx5_action_handle_destroy(dev,
8128                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8129         }
8130         return ret;
8131 }
8132
8133 #ifndef HAVE_MLX5DV_DR
8134 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8135 #else
8136 #define MLX5_DOMAIN_SYNC_FLOW \
8137         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8138 #endif
8139
8140 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8141 {
8142         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8143         const struct mlx5_flow_driver_ops *fops;
8144         int ret;
8145         struct rte_flow_attr attr = { .transfer = 0 };
8146
8147         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8148         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8149         if (ret > 0)
8150                 ret = -ret;
8151         return ret;
8152 }
8153
8154 const struct mlx5_flow_tunnel *
8155 mlx5_get_tof(const struct rte_flow_item *item,
8156              const struct rte_flow_action *action,
8157              enum mlx5_tof_rule_type *rule_type)
8158 {
8159         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8160                 if (item->type == (typeof(item->type))
8161                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8162                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8163                         return flow_items_to_tunnel(item);
8164                 }
8165         }
8166         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8167                 if (action->type == (typeof(action->type))
8168                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8169                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8170                         return flow_actions_to_tunnel(action);
8171                 }
8172         }
8173         return NULL;
8174 }
8175
8176 /**
8177  * tunnel offload functionalilty is defined for DV environment only
8178  */
8179 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8180 __extension__
8181 union tunnel_offload_mark {
8182         uint32_t val;
8183         struct {
8184                 uint32_t app_reserve:8;
8185                 uint32_t table_id:15;
8186                 uint32_t transfer:1;
8187                 uint32_t _unused_:8;
8188         };
8189 };
8190
8191 static bool
8192 mlx5_access_tunnel_offload_db
8193         (struct rte_eth_dev *dev,
8194          bool (*match)(struct rte_eth_dev *,
8195                        struct mlx5_flow_tunnel *, const void *),
8196          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8197          void (*miss)(struct rte_eth_dev *, void *),
8198          void *ctx, bool lock_op);
8199
8200 static int
8201 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8202                              struct rte_flow *flow,
8203                              const struct rte_flow_attr *attr,
8204                              const struct rte_flow_action *app_actions,
8205                              uint32_t flow_idx,
8206                              const struct mlx5_flow_tunnel *tunnel,
8207                              struct tunnel_default_miss_ctx *ctx,
8208                              struct rte_flow_error *error)
8209 {
8210         struct mlx5_priv *priv = dev->data->dev_private;
8211         struct mlx5_flow *dev_flow;
8212         struct rte_flow_attr miss_attr = *attr;
8213         const struct rte_flow_item miss_items[2] = {
8214                 {
8215                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8216                         .spec = NULL,
8217                         .last = NULL,
8218                         .mask = NULL
8219                 },
8220                 {
8221                         .type = RTE_FLOW_ITEM_TYPE_END,
8222                         .spec = NULL,
8223                         .last = NULL,
8224                         .mask = NULL
8225                 }
8226         };
8227         union tunnel_offload_mark mark_id;
8228         struct rte_flow_action_mark miss_mark;
8229         struct rte_flow_action miss_actions[3] = {
8230                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8231                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8232         };
8233         const struct rte_flow_action_jump *jump_data;
8234         uint32_t i, flow_table = 0; /* prevent compilation warning */
8235         struct flow_grp_info grp_info = {
8236                 .external = 1,
8237                 .transfer = attr->transfer,
8238                 .fdb_def_rule = !!priv->fdb_def_rule,
8239                 .std_tbl_fix = 0,
8240         };
8241         int ret;
8242
8243         if (!attr->transfer) {
8244                 uint32_t q_size;
8245
8246                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8247                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8248                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8249                                          0, SOCKET_ID_ANY);
8250                 if (!ctx->queue)
8251                         return rte_flow_error_set
8252                                 (error, ENOMEM,
8253                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8254                                 NULL, "invalid default miss RSS");
8255                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8256                 ctx->action_rss.level = 0,
8257                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8258                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8259                 ctx->action_rss.queue_num = priv->reta_idx_n,
8260                 ctx->action_rss.key = priv->rss_conf.rss_key,
8261                 ctx->action_rss.queue = ctx->queue;
8262                 if (!priv->reta_idx_n || !priv->rxqs_n)
8263                         return rte_flow_error_set
8264                                 (error, EINVAL,
8265                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8266                                 NULL, "invalid port configuration");
8267                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8268                         ctx->action_rss.types = 0;
8269                 for (i = 0; i != priv->reta_idx_n; ++i)
8270                         ctx->queue[i] = (*priv->reta_idx)[i];
8271         } else {
8272                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8273                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8274         }
8275         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8276         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8277         jump_data = app_actions->conf;
8278         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8279         miss_attr.group = jump_data->group;
8280         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8281                                        &flow_table, &grp_info, error);
8282         if (ret)
8283                 return rte_flow_error_set(error, EINVAL,
8284                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8285                                           NULL, "invalid tunnel id");
8286         mark_id.app_reserve = 0;
8287         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8288         mark_id.transfer = !!attr->transfer;
8289         mark_id._unused_ = 0;
8290         miss_mark.id = mark_id.val;
8291         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8292                                     miss_items, miss_actions, flow_idx, error);
8293         if (!dev_flow)
8294                 return -rte_errno;
8295         dev_flow->flow = flow;
8296         dev_flow->external = true;
8297         dev_flow->tunnel = tunnel;
8298         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8299         /* Subflow object was created, we must include one in the list. */
8300         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8301                       dev_flow->handle, next);
8302         DRV_LOG(DEBUG,
8303                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8304                 dev->data->port_id, tunnel->app_tunnel.type,
8305                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8306         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8307                                   miss_actions, error);
8308         if (!ret)
8309                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8310                                                   error);
8311
8312         return ret;
8313 }
8314
8315 static const struct mlx5_flow_tbl_data_entry  *
8316 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8317 {
8318         struct mlx5_priv *priv = dev->data->dev_private;
8319         struct mlx5_dev_ctx_shared *sh = priv->sh;
8320         struct mlx5_hlist_entry *he;
8321         union tunnel_offload_mark mbits = { .val = mark };
8322         union mlx5_flow_tbl_key table_key = {
8323                 {
8324                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8325                         .id = 0,
8326                         .reserved = 0,
8327                         .dummy = 0,
8328                         .is_fdb = !!mbits.transfer,
8329                         .is_egress = 0,
8330                 }
8331         };
8332         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
8333         return he ?
8334                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8335 }
8336
8337 static void
8338 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
8339                                    struct mlx5_hlist_entry *entry)
8340 {
8341         struct mlx5_dev_ctx_shared *sh = list->ctx;
8342         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8343
8344         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8345                         tunnel_flow_tbl_to_id(tte->flow_table));
8346         mlx5_free(tte);
8347 }
8348
8349 static int
8350 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
8351                                   struct mlx5_hlist_entry *entry,
8352                                   uint64_t key, void *cb_ctx __rte_unused)
8353 {
8354         union tunnel_tbl_key tbl = {
8355                 .val = key,
8356         };
8357         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8358
8359         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8360 }
8361
8362 static struct mlx5_hlist_entry *
8363 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
8364                                    void *ctx __rte_unused)
8365 {
8366         struct mlx5_dev_ctx_shared *sh = list->ctx;
8367         struct tunnel_tbl_entry *tte;
8368         union tunnel_tbl_key tbl = {
8369                 .val = key,
8370         };
8371
8372         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8373                           sizeof(*tte), 0,
8374                           SOCKET_ID_ANY);
8375         if (!tte)
8376                 goto err;
8377         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8378                           &tte->flow_table);
8379         if (tte->flow_table >= MLX5_MAX_TABLES) {
8380                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8381                         tte->flow_table);
8382                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8383                                 tte->flow_table);
8384                 goto err;
8385         } else if (!tte->flow_table) {
8386                 goto err;
8387         }
8388         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8389         tte->tunnel_id = tbl.tunnel_id;
8390         tte->group = tbl.group;
8391         return &tte->hash;
8392 err:
8393         if (tte)
8394                 mlx5_free(tte);
8395         return NULL;
8396 }
8397
8398 static uint32_t
8399 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8400                                 const struct mlx5_flow_tunnel *tunnel,
8401                                 uint32_t group, uint32_t *table,
8402                                 struct rte_flow_error *error)
8403 {
8404         struct mlx5_hlist_entry *he;
8405         struct tunnel_tbl_entry *tte;
8406         union tunnel_tbl_key key = {
8407                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8408                 .group = group
8409         };
8410         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8411         struct mlx5_hlist *group_hash;
8412
8413         group_hash = tunnel ? tunnel->groups : thub->groups;
8414         he = mlx5_hlist_register(group_hash, key.val, NULL);
8415         if (!he)
8416                 return rte_flow_error_set(error, EINVAL,
8417                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8418                                           NULL,
8419                                           "tunnel group index not supported");
8420         tte = container_of(he, typeof(*tte), hash);
8421         *table = tte->flow_table;
8422         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8423                 dev->data->port_id, key.tunnel_id, group, *table);
8424         return 0;
8425 }
8426
8427 static void
8428 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8429                       struct mlx5_flow_tunnel *tunnel)
8430 {
8431         struct mlx5_priv *priv = dev->data->dev_private;
8432         struct mlx5_indexed_pool *ipool;
8433
8434         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8435                 dev->data->port_id, tunnel->tunnel_id);
8436         LIST_REMOVE(tunnel, chain);
8437         mlx5_hlist_destroy(tunnel->groups);
8438         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8439         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8440 }
8441
8442 static bool
8443 mlx5_access_tunnel_offload_db
8444         (struct rte_eth_dev *dev,
8445          bool (*match)(struct rte_eth_dev *,
8446                        struct mlx5_flow_tunnel *, const void *),
8447          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8448          void (*miss)(struct rte_eth_dev *, void *),
8449          void *ctx, bool lock_op)
8450 {
8451         bool verdict = false;
8452         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8453         struct mlx5_flow_tunnel *tunnel;
8454
8455         rte_spinlock_lock(&thub->sl);
8456         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8457                 verdict = match(dev, tunnel, (const void *)ctx);
8458                 if (verdict)
8459                         break;
8460         }
8461         if (!lock_op)
8462                 rte_spinlock_unlock(&thub->sl);
8463         if (verdict && hit)
8464                 hit(dev, tunnel, ctx);
8465         if (!verdict && miss)
8466                 miss(dev, ctx);
8467         if (lock_op)
8468                 rte_spinlock_unlock(&thub->sl);
8469
8470         return verdict;
8471 }
8472
8473 struct tunnel_db_find_tunnel_id_ctx {
8474         uint32_t tunnel_id;
8475         struct mlx5_flow_tunnel *tunnel;
8476 };
8477
8478 static bool
8479 find_tunnel_id_match(struct rte_eth_dev *dev,
8480                      struct mlx5_flow_tunnel *tunnel, const void *x)
8481 {
8482         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8483
8484         RTE_SET_USED(dev);
8485         return tunnel->tunnel_id == ctx->tunnel_id;
8486 }
8487
8488 static void
8489 find_tunnel_id_hit(struct rte_eth_dev *dev,
8490                    struct mlx5_flow_tunnel *tunnel, void *x)
8491 {
8492         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8493         RTE_SET_USED(dev);
8494         ctx->tunnel = tunnel;
8495 }
8496
8497 static struct mlx5_flow_tunnel *
8498 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8499 {
8500         struct tunnel_db_find_tunnel_id_ctx ctx = {
8501                 .tunnel_id = id,
8502         };
8503
8504         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8505                                       find_tunnel_id_hit, NULL, &ctx, true);
8506
8507         return ctx.tunnel;
8508 }
8509
8510 static struct mlx5_flow_tunnel *
8511 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8512                           const struct rte_flow_tunnel *app_tunnel)
8513 {
8514         struct mlx5_priv *priv = dev->data->dev_private;
8515         struct mlx5_indexed_pool *ipool;
8516         struct mlx5_flow_tunnel *tunnel;
8517         uint32_t id;
8518
8519         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8520         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8521         if (!tunnel)
8522                 return NULL;
8523         if (id >= MLX5_MAX_TUNNELS) {
8524                 mlx5_ipool_free(ipool, id);
8525                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8526                 return NULL;
8527         }
8528         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
8529                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8530                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8531                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
8532         if (!tunnel->groups) {
8533                 mlx5_ipool_free(ipool, id);
8534                 return NULL;
8535         }
8536         tunnel->groups->ctx = priv->sh;
8537         /* initiate new PMD tunnel */
8538         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8539         tunnel->tunnel_id = id;
8540         tunnel->action.type = (typeof(tunnel->action.type))
8541                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8542         tunnel->action.conf = tunnel;
8543         tunnel->item.type = (typeof(tunnel->item.type))
8544                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8545         tunnel->item.spec = tunnel;
8546         tunnel->item.last = NULL;
8547         tunnel->item.mask = NULL;
8548
8549         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8550                 dev->data->port_id, tunnel->tunnel_id);
8551
8552         return tunnel;
8553 }
8554
8555 struct tunnel_db_get_tunnel_ctx {
8556         const struct rte_flow_tunnel *app_tunnel;
8557         struct mlx5_flow_tunnel *tunnel;
8558 };
8559
8560 static bool get_tunnel_match(struct rte_eth_dev *dev,
8561                              struct mlx5_flow_tunnel *tunnel, const void *x)
8562 {
8563         const struct tunnel_db_get_tunnel_ctx *ctx = x;
8564
8565         RTE_SET_USED(dev);
8566         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
8567                        sizeof(*ctx->app_tunnel));
8568 }
8569
8570 static void get_tunnel_hit(struct rte_eth_dev *dev,
8571                            struct mlx5_flow_tunnel *tunnel, void *x)
8572 {
8573         /* called under tunnel spinlock protection */
8574         struct tunnel_db_get_tunnel_ctx *ctx = x;
8575
8576         RTE_SET_USED(dev);
8577         tunnel->refctn++;
8578         ctx->tunnel = tunnel;
8579 }
8580
8581 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
8582 {
8583         /* called under tunnel spinlock protection */
8584         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8585         struct tunnel_db_get_tunnel_ctx *ctx = x;
8586
8587         rte_spinlock_unlock(&thub->sl);
8588         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
8589         rte_spinlock_lock(&thub->sl);
8590         if (ctx->tunnel) {
8591                 ctx->tunnel->refctn = 1;
8592                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
8593         }
8594 }
8595
8596
8597 static int
8598 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8599                      const struct rte_flow_tunnel *app_tunnel,
8600                      struct mlx5_flow_tunnel **tunnel)
8601 {
8602         struct tunnel_db_get_tunnel_ctx ctx = {
8603                 .app_tunnel = app_tunnel,
8604         };
8605
8606         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
8607                                       get_tunnel_miss, &ctx, true);
8608         *tunnel = ctx.tunnel;
8609         return ctx.tunnel ? 0 : -ENOMEM;
8610 }
8611
8612 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8613 {
8614         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8615
8616         if (!thub)
8617                 return;
8618         if (!LIST_EMPTY(&thub->tunnels))
8619                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
8620         mlx5_hlist_destroy(thub->groups);
8621         mlx5_free(thub);
8622 }
8623
8624 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8625 {
8626         int err;
8627         struct mlx5_flow_tunnel_hub *thub;
8628
8629         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8630                            0, SOCKET_ID_ANY);
8631         if (!thub)
8632                 return -ENOMEM;
8633         LIST_INIT(&thub->tunnels);
8634         rte_spinlock_init(&thub->sl);
8635         thub->groups = mlx5_hlist_create("flow groups",
8636                                          rte_align32pow2(MLX5_MAX_TABLES), 0,
8637                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
8638                                          mlx5_flow_tunnel_grp2tbl_match_cb,
8639                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
8640         if (!thub->groups) {
8641                 err = -rte_errno;
8642                 goto err;
8643         }
8644         thub->groups->ctx = sh;
8645         sh->tunnel_hub = thub;
8646
8647         return 0;
8648
8649 err:
8650         if (thub->groups)
8651                 mlx5_hlist_destroy(thub->groups);
8652         if (thub)
8653                 mlx5_free(thub);
8654         return err;
8655 }
8656
8657 static inline bool
8658 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
8659                           struct rte_flow_tunnel *tunnel,
8660                           const char *err_msg)
8661 {
8662         err_msg = NULL;
8663         if (!is_tunnel_offload_active(dev)) {
8664                 err_msg = "tunnel offload was not activated";
8665                 goto out;
8666         } else if (!tunnel) {
8667                 err_msg = "no application tunnel";
8668                 goto out;
8669         }
8670
8671         switch (tunnel->type) {
8672         default:
8673                 err_msg = "unsupported tunnel type";
8674                 goto out;
8675         case RTE_FLOW_ITEM_TYPE_VXLAN:
8676                 break;
8677         }
8678
8679 out:
8680         return !err_msg;
8681 }
8682
8683 static int
8684 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
8685                     struct rte_flow_tunnel *app_tunnel,
8686                     struct rte_flow_action **actions,
8687                     uint32_t *num_of_actions,
8688                     struct rte_flow_error *error)
8689 {
8690         int ret;
8691         struct mlx5_flow_tunnel *tunnel;
8692         const char *err_msg = NULL;
8693         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8694
8695         if (!verdict)
8696                 return rte_flow_error_set(error, EINVAL,
8697                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8698                                           err_msg);
8699         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8700         if (ret < 0) {
8701                 return rte_flow_error_set(error, ret,
8702                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8703                                           "failed to initialize pmd tunnel");
8704         }
8705         *actions = &tunnel->action;
8706         *num_of_actions = 1;
8707         return 0;
8708 }
8709
8710 static int
8711 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
8712                        struct rte_flow_tunnel *app_tunnel,
8713                        struct rte_flow_item **items,
8714                        uint32_t *num_of_items,
8715                        struct rte_flow_error *error)
8716 {
8717         int ret;
8718         struct mlx5_flow_tunnel *tunnel;
8719         const char *err_msg = NULL;
8720         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8721
8722         if (!verdict)
8723                 return rte_flow_error_set(error, EINVAL,
8724                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8725                                           err_msg);
8726         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8727         if (ret < 0) {
8728                 return rte_flow_error_set(error, ret,
8729                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8730                                           "failed to initialize pmd tunnel");
8731         }
8732         *items = &tunnel->item;
8733         *num_of_items = 1;
8734         return 0;
8735 }
8736
8737 struct tunnel_db_element_release_ctx {
8738         struct rte_flow_item *items;
8739         struct rte_flow_action *actions;
8740         uint32_t num_elements;
8741         struct rte_flow_error *error;
8742         int ret;
8743 };
8744
8745 static bool
8746 tunnel_element_release_match(struct rte_eth_dev *dev,
8747                              struct mlx5_flow_tunnel *tunnel, const void *x)
8748 {
8749         const struct tunnel_db_element_release_ctx *ctx = x;
8750
8751         RTE_SET_USED(dev);
8752         if (ctx->num_elements != 1)
8753                 return false;
8754         else if (ctx->items)
8755                 return ctx->items == &tunnel->item;
8756         else if (ctx->actions)
8757                 return ctx->actions == &tunnel->action;
8758
8759         return false;
8760 }
8761
8762 static void
8763 tunnel_element_release_hit(struct rte_eth_dev *dev,
8764                            struct mlx5_flow_tunnel *tunnel, void *x)
8765 {
8766         struct tunnel_db_element_release_ctx *ctx = x;
8767         ctx->ret = 0;
8768         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
8769                 mlx5_flow_tunnel_free(dev, tunnel);
8770 }
8771
8772 static void
8773 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
8774 {
8775         struct tunnel_db_element_release_ctx *ctx = x;
8776         RTE_SET_USED(dev);
8777         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
8778                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8779                                       "invalid argument");
8780 }
8781
8782 static int
8783 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
8784                        struct rte_flow_item *pmd_items,
8785                        uint32_t num_items, struct rte_flow_error *err)
8786 {
8787         struct tunnel_db_element_release_ctx ctx = {
8788                 .items = pmd_items,
8789                 .actions = NULL,
8790                 .num_elements = num_items,
8791                 .error = err,
8792         };
8793
8794         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8795                                       tunnel_element_release_hit,
8796                                       tunnel_element_release_miss, &ctx, false);
8797
8798         return ctx.ret;
8799 }
8800
8801 static int
8802 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
8803                          struct rte_flow_action *pmd_actions,
8804                          uint32_t num_actions, struct rte_flow_error *err)
8805 {
8806         struct tunnel_db_element_release_ctx ctx = {
8807                 .items = NULL,
8808                 .actions = pmd_actions,
8809                 .num_elements = num_actions,
8810                 .error = err,
8811         };
8812
8813         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8814                                       tunnel_element_release_hit,
8815                                       tunnel_element_release_miss, &ctx, false);
8816
8817         return ctx.ret;
8818 }
8819
8820 static int
8821 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
8822                                   struct rte_mbuf *m,
8823                                   struct rte_flow_restore_info *info,
8824                                   struct rte_flow_error *err)
8825 {
8826         uint64_t ol_flags = m->ol_flags;
8827         const struct mlx5_flow_tbl_data_entry *tble;
8828         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
8829
8830         if (!is_tunnel_offload_active(dev)) {
8831                 info->flags = 0;
8832                 return 0;
8833         }
8834
8835         if ((ol_flags & mask) != mask)
8836                 goto err;
8837         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
8838         if (!tble) {
8839                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
8840                         dev->data->port_id, m->hash.fdir.hi);
8841                 goto err;
8842         }
8843         MLX5_ASSERT(tble->tunnel);
8844         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
8845         info->group_id = tble->group_id;
8846         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
8847                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
8848                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
8849
8850         return 0;
8851
8852 err:
8853         return rte_flow_error_set(err, EINVAL,
8854                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8855                                   "failed to get restore info");
8856 }
8857
8858 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
8859 static int
8860 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
8861                            __rte_unused struct rte_flow_tunnel *app_tunnel,
8862                            __rte_unused struct rte_flow_action **actions,
8863                            __rte_unused uint32_t *num_of_actions,
8864                            __rte_unused struct rte_flow_error *error)
8865 {
8866         return -ENOTSUP;
8867 }
8868
8869 static int
8870 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
8871                        __rte_unused struct rte_flow_tunnel *app_tunnel,
8872                        __rte_unused struct rte_flow_item **items,
8873                        __rte_unused uint32_t *num_of_items,
8874                        __rte_unused struct rte_flow_error *error)
8875 {
8876         return -ENOTSUP;
8877 }
8878
8879 static int
8880 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
8881                               __rte_unused struct rte_flow_item *pmd_items,
8882                               __rte_unused uint32_t num_items,
8883                               __rte_unused struct rte_flow_error *err)
8884 {
8885         return -ENOTSUP;
8886 }
8887
8888 static int
8889 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
8890                                 __rte_unused struct rte_flow_action *pmd_action,
8891                                 __rte_unused uint32_t num_actions,
8892                                 __rte_unused struct rte_flow_error *err)
8893 {
8894         return -ENOTSUP;
8895 }
8896
8897 static int
8898 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
8899                                   __rte_unused struct rte_mbuf *m,
8900                                   __rte_unused struct rte_flow_restore_info *i,
8901                                   __rte_unused struct rte_flow_error *err)
8902 {
8903         return -ENOTSUP;
8904 }
8905
8906 static int
8907 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
8908                              __rte_unused struct rte_flow *flow,
8909                              __rte_unused const struct rte_flow_attr *attr,
8910                              __rte_unused const struct rte_flow_action *actions,
8911                              __rte_unused uint32_t flow_idx,
8912                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
8913                              __rte_unused struct tunnel_default_miss_ctx *ctx,
8914                              __rte_unused struct rte_flow_error *error)
8915 {
8916         return -ENOTSUP;
8917 }
8918
8919 static struct mlx5_flow_tunnel *
8920 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
8921                     __rte_unused uint32_t id)
8922 {
8923         return NULL;
8924 }
8925
8926 static void
8927 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
8928                       __rte_unused struct mlx5_flow_tunnel *tunnel)
8929 {
8930 }
8931
8932 static uint32_t
8933 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
8934                                 __rte_unused const struct mlx5_flow_tunnel *t,
8935                                 __rte_unused uint32_t group,
8936                                 __rte_unused uint32_t *table,
8937                                 struct rte_flow_error *error)
8938 {
8939         return rte_flow_error_set(error, ENOTSUP,
8940                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8941                                   "tunnel offload requires DV support");
8942 }
8943
8944 void
8945 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
8946                         __rte_unused  uint16_t port_id)
8947 {
8948 }
8949 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */