drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint8_t optional;
 104         /**< optional expand field. Default 0 to expand, 1 not go deeper. */
 105 };
 106
 107 /** Object returned by mlx5_flow_expand_rss(). */
 108 struct mlx5_flow_expand_rss {
 109         uint32_t entries;
 110         /**< Number of entries @p patterns and @p priorities. */
 111         struct {
 112                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 113                 uint32_t priority; /**< Priority offset for each expansion. */
 114         } entry[];
 115 };
 116
 117 static enum rte_flow_item_type
 118 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 119 {
 120         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 121         uint16_t ether_type = 0;
 122         uint16_t ether_type_m;
 123         uint8_t ip_next_proto = 0;
 124         uint8_t ip_next_proto_m;
 125
 126         if (item == NULL || item->spec == NULL)
 127                 return ret;
 128         switch (item->type) {
 129         case RTE_FLOW_ITEM_TYPE_ETH:
 130                 if (item->mask)
 131                         ether_type_m = ((const struct rte_flow_item_eth *)
 132                                                 (item->mask))->type;
 133                 else
 134                         ether_type_m = rte_flow_item_eth_mask.type;
 135                 if (ether_type_m != RTE_BE16(0xFFFF))
 136                         break;
 137                 ether_type = ((const struct rte_flow_item_eth *)
 138                                 (item->spec))->type;
 139                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 140                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 141                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 142                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 143                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 144                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 145                 else
 146                         ret = RTE_FLOW_ITEM_TYPE_END;
 147                 break;
 148         case RTE_FLOW_ITEM_TYPE_VLAN:
 149                 if (item->mask)
 150                         ether_type_m = ((const struct rte_flow_item_vlan *)
 151                                                 (item->mask))->inner_type;
 152                 else
 153                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 154                 if (ether_type_m != RTE_BE16(0xFFFF))
 155                         break;
 156                 ether_type = ((const struct rte_flow_item_vlan *)
 157                                 (item->spec))->inner_type;
 158                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 159                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 160                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 161                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 162                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 163                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 164                 else
 165                         ret = RTE_FLOW_ITEM_TYPE_END;
 166                 break;
 167         case RTE_FLOW_ITEM_TYPE_IPV4:
 168                 if (item->mask)
 169                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 170                                         (item->mask))->hdr.next_proto_id;
 171                 else
 172                         ip_next_proto_m =
 173                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 174                 if (ip_next_proto_m != 0xFF)
 175                         break;
 176                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 177                                 (item->spec))->hdr.next_proto_id;
 178                 if (ip_next_proto == IPPROTO_UDP)
 179                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 180                 else if (ip_next_proto == IPPROTO_TCP)
 181                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 182                 else if (ip_next_proto == IPPROTO_IP)
 183                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 else if (ip_next_proto == IPPROTO_IPV6)
 185                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 186                 else
 187                         ret = RTE_FLOW_ITEM_TYPE_END;
 188                 break;
 189         case RTE_FLOW_ITEM_TYPE_IPV6:
 190                 if (item->mask)
 191                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 192                                                 (item->mask))->hdr.proto;
 193                 else
 194                         ip_next_proto_m =
 195                                 rte_flow_item_ipv6_mask.hdr.proto;
 196                 if (ip_next_proto_m != 0xFF)
 197                         break;
 198                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 199                                 (item->spec))->hdr.proto;
 200                 if (ip_next_proto == IPPROTO_UDP)
 201                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 202                 else if (ip_next_proto == IPPROTO_TCP)
 203                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 204                 else if (ip_next_proto == IPPROTO_IP)
 205                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 206                 else if (ip_next_proto == IPPROTO_IPV6)
 207                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 208                 else
 209                         ret = RTE_FLOW_ITEM_TYPE_END;
 210                 break;
 211         default:
 212                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 213                 break;
 214         }
 215         return ret;
 216 }
 217
 218 #define MLX5_RSS_EXP_ELT_N 16
 219
 220 /**
 221  * Expand RSS flows into several possible flows according to the RSS hash
 222  * fields requested and the driver capabilities.
 223  *
 224  * @param[out] buf
 225  *   Buffer to store the result expansion.
 226  * @param[in] size
 227  *   Buffer size in bytes. If 0, @p buf can be NULL.
 228  * @param[in] pattern
 229  *   User flow pattern.
 230  * @param[in] types
 231  *   RSS types to expand (see ETH_RSS_* definitions).
 232  * @param[in] graph
 233  *   Input graph to expand @p pattern according to @p types.
 234  * @param[in] graph_root_index
 235  *   Index of root node in @p graph, typically 0.
 236  *
 237  * @return
 238  *   A positive value representing the size of @p buf in bytes regardless of
 239  *   @p size on success, a negative errno value otherwise and rte_errno is
 240  *   set, the following errors are defined:
 241  *
 242  *   -E2BIG: graph-depth @p graph is too deep.
 243  */
 244 static int
 245 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 246                      const struct rte_flow_item *pattern, uint64_t types,
 247                      const struct mlx5_flow_expand_node graph[],
 248                      int graph_root_index)
 249 {
 250         const struct rte_flow_item *item;
 251         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 252         const int *next_node;
 253         const int *stack[MLX5_RSS_EXP_ELT_N];
 254         int stack_pos = 0;
 255         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 256         unsigned int i;
 257         size_t lsize;
 258         size_t user_pattern_size = 0;
 259         void *addr = NULL;
 260         const struct mlx5_flow_expand_node *next = NULL;
 261         struct rte_flow_item missed_item;
 262         int missed = 0;
 263         int elt = 0;
 264         const struct rte_flow_item *last_item = NULL;
 265
 266         memset(&missed_item, 0, sizeof(missed_item));
 267         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 268                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 269         if (lsize <= size) {
 270                 buf->entry[0].priority = 0;
 271                 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 272                 buf->entries = 0;
 273                 addr = buf->entry[0].pattern;
 274         }
 275         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 277                         last_item = item;
 278                 for (i = 0; node->next && node->next[i]; ++i) {
 279                         next = &graph[node->next[i]];
 280                         if (next->type == item->type)
 281                                 break;
 282                 }
 283                 if (next)
 284                         node = next;
 285                 user_pattern_size += sizeof(*item);
 286         }
 287         user_pattern_size += sizeof(*item); /* Handle END item. */
 288         lsize += user_pattern_size;
 289         /* Copy the user pattern in the first entry of the buffer. */
 290         if (lsize <= size) {
 291                 rte_memcpy(addr, pattern, user_pattern_size);
 292                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 293                 buf->entries = 1;
 294         }
 295         /* Start expanding. */
 296         memset(flow_items, 0, sizeof(flow_items));
 297         user_pattern_size -= sizeof(*item);
 298         /*
 299          * Check if the last valid item has spec set, need complete pattern,
 300          * and the pattern can be used for expansion.
 301          */
 302         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 303         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 304                 /* Item type END indicates expansion is not required. */
 305                 return lsize;
 306         }
 307         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 308                 next = NULL;
 309                 missed = 1;
 310                 for (i = 0; node->next && node->next[i]; ++i) {
 311                         next = &graph[node->next[i]];
 312                         if (next->type == missed_item.type) {
 313                                 flow_items[0].type = missed_item.type;
 314                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 315                                 break;
 316                         }
 317                         next = NULL;
 318                 }
 319         }
 320         if (next && missed) {
 321                 elt = 2; /* missed item + item end. */
 322                 node = next;
 323                 lsize += elt * sizeof(*item) + user_pattern_size;
 324                 if ((node->rss_types & types) && lsize <= size) {
 325                         buf->entry[buf->entries].priority = 1;
 326                         buf->entry[buf->entries].pattern = addr;
 327                         buf->entries++;
 328                         rte_memcpy(addr, buf->entry[0].pattern,
 329                                    user_pattern_size);
 330                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 331                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 332                         addr = (void *)(((uintptr_t)addr) +
 333                                         elt * sizeof(*item));
 334                 }
 335         }
 336         memset(flow_items, 0, sizeof(flow_items));
 337         next_node = node->next;
 338         stack[stack_pos] = next_node;
 339         node = next_node ? &graph[*next_node] : NULL;
 340         while (node) {
 341                 flow_items[stack_pos].type = node->type;
 342                 if (node->rss_types & types) {
 343                         /*
 344                          * compute the number of items to copy from the
 345                          * expansion and copy it.
 346                          * When the stack_pos is 0, there are 1 element in it,
 347                          * plus the addition END item.
 348                          */
 349                         elt = stack_pos + 2;
 350                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 351                         lsize += elt * sizeof(*item) + user_pattern_size;
 352                         if (lsize <= size) {
 353                                 size_t n = elt * sizeof(*item);
 354
 355                                 buf->entry[buf->entries].priority =
 356                                         stack_pos + 1 + missed;
 357                                 buf->entry[buf->entries].pattern = addr;
 358                                 buf->entries++;
 359                                 rte_memcpy(addr, buf->entry[0].pattern,
 360                                            user_pattern_size);
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                                 user_pattern_size);
 363                                 rte_memcpy(addr, &missed_item,
 364                                            missed * sizeof(*item));
 365                                 addr = (void *)(((uintptr_t)addr) +
 366                                         missed * sizeof(*item));
 367                                 rte_memcpy(addr, flow_items, n);
 368                                 addr = (void *)(((uintptr_t)addr) + n);
 369                         }
 370                 }
 371                 /* Go deeper. */
 372                 if (!node->optional && node->next) {
 373                         next_node = node->next;
 374                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 375                                 rte_errno = E2BIG;
 376                                 return -rte_errno;
 377                         }
 378                         stack[stack_pos] = next_node;
 379                 } else if (*(next_node + 1)) {
 380                         /* Follow up with the next possibility. */
 381                         ++next_node;
 382                 } else {
 383                         /* Move to the next path. */
 384                         if (stack_pos)
 385                                 next_node = stack[--stack_pos];
 386                         next_node++;
 387                         stack[stack_pos] = next_node;
 388                 }
 389                 node = *next_node ? &graph[*next_node] : NULL;
 390         };
 391         return lsize;
 392 }
 393
 394 enum mlx5_expansion {
 395         MLX5_EXPANSION_ROOT,
 396         MLX5_EXPANSION_ROOT_OUTER,
 397         MLX5_EXPANSION_ROOT_ETH_VLAN,
 398         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 399         MLX5_EXPANSION_OUTER_ETH,
 400         MLX5_EXPANSION_OUTER_ETH_VLAN,
 401         MLX5_EXPANSION_OUTER_VLAN,
 402         MLX5_EXPANSION_OUTER_IPV4,
 403         MLX5_EXPANSION_OUTER_IPV4_UDP,
 404         MLX5_EXPANSION_OUTER_IPV4_TCP,
 405         MLX5_EXPANSION_OUTER_IPV6,
 406         MLX5_EXPANSION_OUTER_IPV6_UDP,
 407         MLX5_EXPANSION_OUTER_IPV6_TCP,
 408         MLX5_EXPANSION_VXLAN,
 409         MLX5_EXPANSION_VXLAN_GPE,
 410         MLX5_EXPANSION_GRE,
 411         MLX5_EXPANSION_GRE_KEY,
 412         MLX5_EXPANSION_MPLS,
 413         MLX5_EXPANSION_ETH,
 414         MLX5_EXPANSION_ETH_VLAN,
 415         MLX5_EXPANSION_VLAN,
 416         MLX5_EXPANSION_IPV4,
 417         MLX5_EXPANSION_IPV4_UDP,
 418         MLX5_EXPANSION_IPV4_TCP,
 419         MLX5_EXPANSION_IPV6,
 420         MLX5_EXPANSION_IPV6_UDP,
 421         MLX5_EXPANSION_IPV6_TCP,
 422 };
 423
 424 /** Supported expansion of items. */
 425 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 426         [MLX5_EXPANSION_ROOT] = {
 427                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 428                                                   MLX5_EXPANSION_IPV4,
 429                                                   MLX5_EXPANSION_IPV6),
 430                 .type = RTE_FLOW_ITEM_TYPE_END,
 431         },
 432         [MLX5_EXPANSION_ROOT_OUTER] = {
 433                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 434                                                   MLX5_EXPANSION_OUTER_IPV4,
 435                                                   MLX5_EXPANSION_OUTER_IPV6),
 436                 .type = RTE_FLOW_ITEM_TYPE_END,
 437         },
 438         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 439                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 440                 .type = RTE_FLOW_ITEM_TYPE_END,
 441         },
 442         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 443                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 444                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 445                 .type = RTE_FLOW_ITEM_TYPE_END,
 446         },
 447         [MLX5_EXPANSION_OUTER_ETH] = {
 448                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 449                                                   MLX5_EXPANSION_OUTER_IPV6,
 450                                                   MLX5_EXPANSION_MPLS),
 451                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 452                 .rss_types = 0,
 453         },
 454         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 455                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 457                 .rss_types = 0,
 458         },
 459         [MLX5_EXPANSION_OUTER_VLAN] = {
 460                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 461                                                   MLX5_EXPANSION_OUTER_IPV6),
 462                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 463         },
 464         [MLX5_EXPANSION_OUTER_IPV4] = {
 465                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 466                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 467                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 468                          MLX5_EXPANSION_GRE,
 469                          MLX5_EXPANSION_IPV4,
 470                          MLX5_EXPANSION_IPV6),
 471                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 472                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 473                         ETH_RSS_NONFRAG_IPV4_OTHER,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 477                                                   MLX5_EXPANSION_VXLAN_GPE),
 478                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 479                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 480         },
 481         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 482                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 483                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 484         },
 485         [MLX5_EXPANSION_OUTER_IPV6] = {
 486                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 487                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 488                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 489                          MLX5_EXPANSION_IPV4,
 490                          MLX5_EXPANSION_IPV6,
 491                          MLX5_EXPANSION_GRE),
 492                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 493                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 494                         ETH_RSS_NONFRAG_IPV6_OTHER,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 498                                                   MLX5_EXPANSION_VXLAN_GPE),
 499                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 500                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 501         },
 502         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 503                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 504                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 505         },
 506         [MLX5_EXPANSION_VXLAN] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 508                                                   MLX5_EXPANSION_IPV4,
 509                                                   MLX5_EXPANSION_IPV6),
 510                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 511         },
 512         [MLX5_EXPANSION_VXLAN_GPE] = {
 513                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 514                                                   MLX5_EXPANSION_IPV4,
 515                                                   MLX5_EXPANSION_IPV6),
 516                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 517         },
 518         [MLX5_EXPANSION_GRE] = {
 519                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 520                                                   MLX5_EXPANSION_IPV6,
 521                                                   MLX5_EXPANSION_GRE_KEY),
 522                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 523         },
 524         [MLX5_EXPANSION_GRE_KEY] = {
 525                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 526                                                   MLX5_EXPANSION_IPV6),
 527                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 528                 .optional = 1,
 529         },
 530         [MLX5_EXPANSION_MPLS] = {
 531                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 532                                                   MLX5_EXPANSION_IPV6),
 533                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 534         },
 535         [MLX5_EXPANSION_ETH] = {
 536                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 537                                                   MLX5_EXPANSION_IPV6),
 538                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 539         },
 540         [MLX5_EXPANSION_ETH_VLAN] = {
 541                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 542                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 543         },
 544         [MLX5_EXPANSION_VLAN] = {
 545                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 546                                                   MLX5_EXPANSION_IPV6),
 547                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 548         },
 549         [MLX5_EXPANSION_IPV4] = {
 550                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 551                                                   MLX5_EXPANSION_IPV4_TCP),
 552                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 553                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 554                         ETH_RSS_NONFRAG_IPV4_OTHER,
 555         },
 556         [MLX5_EXPANSION_IPV4_UDP] = {
 557                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 558                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 559         },
 560         [MLX5_EXPANSION_IPV4_TCP] = {
 561                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 562                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 563         },
 564         [MLX5_EXPANSION_IPV6] = {
 565                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 566                                                   MLX5_EXPANSION_IPV6_TCP),
 567                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 568                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 569                         ETH_RSS_NONFRAG_IPV6_OTHER,
 570         },
 571         [MLX5_EXPANSION_IPV6_UDP] = {
 572                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 573                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 574         },
 575         [MLX5_EXPANSION_IPV6_TCP] = {
 576                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 577                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 578         },
 579 };
 580
 581 static struct rte_flow_action_handle *
 582 mlx5_action_handle_create(struct rte_eth_dev *dev,
 583                           const struct rte_flow_indir_action_conf *conf,
 584                           const struct rte_flow_action *action,
 585                           struct rte_flow_error *error);
 586 static int mlx5_action_handle_destroy
 587                                 (struct rte_eth_dev *dev,
 588                                  struct rte_flow_action_handle *handle,
 589                                  struct rte_flow_error *error);
 590 static int mlx5_action_handle_update
 591                                 (struct rte_eth_dev *dev,
 592                                  struct rte_flow_action_handle *handle,
 593                                  const void *update,
 594                                  struct rte_flow_error *error);
 595 static int mlx5_action_handle_query
 596                                 (struct rte_eth_dev *dev,
 597                                  const struct rte_flow_action_handle *handle,
 598                                  void *data,
 599                                  struct rte_flow_error *error);
 600 static int
 601 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 602                     struct rte_flow_tunnel *app_tunnel,
 603                     struct rte_flow_action **actions,
 604                     uint32_t *num_of_actions,
 605                     struct rte_flow_error *error);
 606 static int
 607 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 608                        struct rte_flow_tunnel *app_tunnel,
 609                        struct rte_flow_item **items,
 610                        uint32_t *num_of_items,
 611                        struct rte_flow_error *error);
 612 static int
 613 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 614                               struct rte_flow_item *pmd_items,
 615                               uint32_t num_items, struct rte_flow_error *err);
 616 static int
 617 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 618                                 struct rte_flow_action *pmd_actions,
 619                                 uint32_t num_actions,
 620                                 struct rte_flow_error *err);
 621 static int
 622 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 623                                   struct rte_mbuf *m,
 624                                   struct rte_flow_restore_info *info,
 625                                   struct rte_flow_error *err);
 626
 627 static const struct rte_flow_ops mlx5_flow_ops = {
 628         .validate = mlx5_flow_validate,
 629         .create = mlx5_flow_create,
 630         .destroy = mlx5_flow_destroy,
 631         .flush = mlx5_flow_flush,
 632         .isolate = mlx5_flow_isolate,
 633         .query = mlx5_flow_query,
 634         .dev_dump = mlx5_flow_dev_dump,
 635         .get_aged_flows = mlx5_flow_get_aged_flows,
 636         .action_handle_create = mlx5_action_handle_create,
 637         .action_handle_destroy = mlx5_action_handle_destroy,
 638         .action_handle_update = mlx5_action_handle_update,
 639         .action_handle_query = mlx5_action_handle_query,
 640         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 641         .tunnel_match = mlx5_flow_tunnel_match,
 642         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 643         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 644         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 645 };
 646
 647 /* Tunnel information. */
 648 struct mlx5_flow_tunnel_info {
 649         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 650         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 651 };
 652
 653 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 654         {
 655                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 656                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 657         },
 658         {
 659                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 660                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 661         },
 662         {
 663                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 664                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 665         },
 666         {
 667                 .tunnel = MLX5_FLOW_LAYER_GRE,
 668                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 669         },
 670         {
 671                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 672                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 673         },
 674         {
 675                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 676                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 677         },
 678         {
 679                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 680                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 681         },
 682         {
 683                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 684                 .ptype = RTE_PTYPE_TUNNEL_IP,
 685         },
 686         {
 687                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 688                 .ptype = RTE_PTYPE_TUNNEL_IP,
 689         },
 690         {
 691                 .tunnel = MLX5_FLOW_LAYER_GTP,
 692                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 693         },
 694 };
 695
 696
 697
 698 /**
 699  * Translate tag ID to register.
 700  *
 701  * @param[in] dev
 702  *   Pointer to the Ethernet device structure.
 703  * @param[in] feature
 704  *   The feature that request the register.
 705  * @param[in] id
 706  *   The request register ID.
 707  * @param[out] error
 708  *   Error description in case of any.
 709  *
 710  * @return
 711  *   The request register on success, a negative errno
 712  *   value otherwise and rte_errno is set.
 713  */
 714 int
 715 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 716                      enum mlx5_feature_name feature,
 717                      uint32_t id,
 718                      struct rte_flow_error *error)
 719 {
 720         struct mlx5_priv *priv = dev->data->dev_private;
 721         struct mlx5_dev_config *config = &priv->config;
 722         enum modify_reg start_reg;
 723         bool skip_mtr_reg = false;
 724
 725         switch (feature) {
 726         case MLX5_HAIRPIN_RX:
 727                 return REG_B;
 728         case MLX5_HAIRPIN_TX:
 729                 return REG_A;
 730         case MLX5_METADATA_RX:
 731                 switch (config->dv_xmeta_en) {
 732                 case MLX5_XMETA_MODE_LEGACY:
 733                         return REG_B;
 734                 case MLX5_XMETA_MODE_META16:
 735                         return REG_C_0;
 736                 case MLX5_XMETA_MODE_META32:
 737                         return REG_C_1;
 738                 }
 739                 break;
 740         case MLX5_METADATA_TX:
 741                 return REG_A;
 742         case MLX5_METADATA_FDB:
 743                 switch (config->dv_xmeta_en) {
 744                 case MLX5_XMETA_MODE_LEGACY:
 745                         return REG_NON;
 746                 case MLX5_XMETA_MODE_META16:
 747                         return REG_C_0;
 748                 case MLX5_XMETA_MODE_META32:
 749                         return REG_C_1;
 750                 }
 751                 break;
 752         case MLX5_FLOW_MARK:
 753                 switch (config->dv_xmeta_en) {
 754                 case MLX5_XMETA_MODE_LEGACY:
 755                         return REG_NON;
 756                 case MLX5_XMETA_MODE_META16:
 757                         return REG_C_1;
 758                 case MLX5_XMETA_MODE_META32:
 759                         return REG_C_0;
 760                 }
 761                 break;
 762         case MLX5_MTR_ID:
 763                 /*
 764                  * If meter color and meter id share one register, flow match
 765                  * should use the meter color register for match.
 766                  */
 767                 if (priv->mtr_reg_share)
 768                         return priv->mtr_color_reg;
 769                 else
 770                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 771                                REG_C_3;
 772         case MLX5_MTR_COLOR:
 773         case MLX5_ASO_FLOW_HIT:
 774         case MLX5_ASO_CONNTRACK:
 775                 /* All features use the same REG_C. */
 776                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 777                 return priv->mtr_color_reg;
 778         case MLX5_COPY_MARK:
 779                 /*
 780                  * Metadata COPY_MARK register using is in meter suffix sub
 781                  * flow while with meter. It's safe to share the same register.
 782                  */
 783                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 784         case MLX5_APP_TAG:
 785                 /*
 786                  * If meter is enable, it will engage the register for color
 787                  * match and flow match. If meter color match is not using the
 788                  * REG_C_2, need to skip the REG_C_x be used by meter color
 789                  * match.
 790                  * If meter is disable, free to use all available registers.
 791                  */
 792                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 793                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 794                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 795                 if (id > (uint32_t)(REG_C_7 - start_reg))
 796                         return rte_flow_error_set(error, EINVAL,
 797                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 798                                                   NULL, "invalid tag id");
 799                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 800                         return rte_flow_error_set(error, ENOTSUP,
 801                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 802                                                   NULL, "unsupported tag id");
 803                 /*
 804                  * This case means meter is using the REG_C_x great than 2.
 805                  * Take care not to conflict with meter color REG_C_x.
 806                  * If the available index REG_C_y >= REG_C_x, skip the
 807                  * color register.
 808                  */
 809                 if (skip_mtr_reg && config->flow_mreg_c
 810                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 811                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 812                                 return rte_flow_error_set(error, EINVAL,
 813                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 814                                                         NULL, "invalid tag id");
 815                         if (config->flow_mreg_c
 816                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 817                                 return config->flow_mreg_c
 818                                                [id + 1 + start_reg - REG_C_0];
 819                         return rte_flow_error_set(error, ENOTSUP,
 820                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 821                                                   NULL, "unsupported tag id");
 822                 }
 823                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 824         }
 825         MLX5_ASSERT(false);
 826         return rte_flow_error_set(error, EINVAL,
 827                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 828                                   NULL, "invalid feature name");
 829 }
 830
 831 /**
 832  * Check extensive flow metadata register support.
 833  *
 834  * @param dev
 835  *   Pointer to rte_eth_dev structure.
 836  *
 837  * @return
 838  *   True if device supports extensive flow metadata register, otherwise false.
 839  */
 840 bool
 841 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 842 {
 843         struct mlx5_priv *priv = dev->data->dev_private;
 844         struct mlx5_dev_config *config = &priv->config;
 845
 846         /*
 847          * Having available reg_c can be regarded inclusively as supporting
 848          * extensive flow metadata register, which could mean,
 849          * - metadata register copy action by modify header.
 850          * - 16 modify header actions is supported.
 851          * - reg_c's are preserved across different domain (FDB and NIC) on
 852          *   packet loopback by flow lookup miss.
 853          */
 854         return config->flow_mreg_c[2] != REG_NON;
 855 }
 856
 857 /**
 858  * Get the lowest priority.
 859  *
 860  * @param[in] dev
 861  *   Pointer to the Ethernet device structure.
 862  * @param[in] attributes
 863  *   Pointer to device flow rule attributes.
 864  *
 865  * @return
 866  *   The value of lowest priority of flow.
 867  */
 868 uint32_t
 869 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 870                           const struct rte_flow_attr *attr)
 871 {
 872         struct mlx5_priv *priv = dev->data->dev_private;
 873
 874         if (!attr->group && !attr->transfer)
 875                 return priv->config.flow_prio - 2;
 876         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 877 }
 878
 879 /**
 880  * Calculate matcher priority of the flow.
 881  *
 882  * @param[in] dev
 883  *   Pointer to the Ethernet device structure.
 884  * @param[in] attr
 885  *   Pointer to device flow rule attributes.
 886  * @param[in] subpriority
 887  *   The priority based on the items.
 888  * @return
 889  *   The matcher priority of the flow.
 890  */
 891 uint16_t
 892 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 893                           const struct rte_flow_attr *attr,
 894                           uint32_t subpriority)
 895 {
 896         uint16_t priority = (uint16_t)attr->priority;
 897         struct mlx5_priv *priv = dev->data->dev_private;
 898
 899         if (!attr->group && !attr->transfer) {
 900                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 901                         priority = priv->config.flow_prio - 1;
 902                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 903         }
 904         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 905                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 906         return priority * 3 + subpriority;
 907 }
 908
 909 /**
 910  * Verify the @p item specifications (spec, last, mask) are compatible with the
 911  * NIC capabilities.
 912  *
 913  * @param[in] item
 914  *   Item specification.
 915  * @param[in] mask
 916  *   @p item->mask or flow default bit-masks.
 917  * @param[in] nic_mask
 918  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 919  * @param[in] size
 920  *   Bit-masks size in bytes.
 921  * @param[in] range_accepted
 922  *   True if range of values is accepted for specific fields, false otherwise.
 923  * @param[out] error
 924  *   Pointer to error structure.
 925  *
 926  * @return
 927  *   0 on success, a negative errno value otherwise and rte_errno is set.
 928  */
 929 int
 930 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 931                           const uint8_t *mask,
 932                           const uint8_t *nic_mask,
 933                           unsigned int size,
 934                           bool range_accepted,
 935                           struct rte_flow_error *error)
 936 {
 937         unsigned int i;
 938
 939         MLX5_ASSERT(nic_mask);
 940         for (i = 0; i < size; ++i)
 941                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 942                         return rte_flow_error_set(error, ENOTSUP,
 943                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 944                                                   item,
 945                                                   "mask enables non supported"
 946                                                   " bits");
 947         if (!item->spec && (item->mask || item->last))
 948                 return rte_flow_error_set(error, EINVAL,
 949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 950                                           "mask/last without a spec is not"
 951                                           " supported");
 952         if (item->spec && item->last && !range_accepted) {
 953                 uint8_t spec[size];
 954                 uint8_t last[size];
 955                 unsigned int i;
 956                 int ret;
 957
 958                 for (i = 0; i < size; ++i) {
 959                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 960                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 961                 }
 962                 ret = memcmp(spec, last, size);
 963                 if (ret != 0)
 964                         return rte_flow_error_set(error, EINVAL,
 965                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 966                                                   item,
 967                                                   "range is not valid");
 968         }
 969         return 0;
 970 }
 971
 972 /**
 973  * Adjust the hash fields according to the @p flow information.
 974  *
 975  * @param[in] dev_flow.
 976  *   Pointer to the mlx5_flow.
 977  * @param[in] tunnel
 978  *   1 when the hash field is for a tunnel item.
 979  * @param[in] layer_types
 980  *   ETH_RSS_* types.
 981  * @param[in] hash_fields
 982  *   Item hash fields.
 983  *
 984  * @return
 985  *   The hash fields that should be used.
 986  */
 987 uint64_t
 988 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 989                             int tunnel __rte_unused, uint64_t layer_types,
 990                             uint64_t hash_fields)
 991 {
 992 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 993         int rss_request_inner = rss_desc->level >= 2;
 994
 995         /* Check RSS hash level for tunnel. */
 996         if (tunnel && rss_request_inner)
 997                 hash_fields |= IBV_RX_HASH_INNER;
 998         else if (tunnel || rss_request_inner)
 999                 return 0;
1000 #endif
1001         /* Check if requested layer matches RSS hash fields. */
1002         if (!(rss_desc->types & layer_types))
1003                 return 0;
1004         return hash_fields;
1005 }
1006
1007 /**
1008  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1009  * if several tunnel rules are used on this queue, the tunnel ptype will be
1010  * cleared.
1011  *
1012  * @param rxq_ctrl
1013  *   Rx queue to update.
1014  */
1015 static void
1016 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1017 {
1018         unsigned int i;
1019         uint32_t tunnel_ptype = 0;
1020
1021         /* Look up for the ptype to use. */
1022         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1023                 if (!rxq_ctrl->flow_tunnels_n[i])
1024                         continue;
1025                 if (!tunnel_ptype) {
1026                         tunnel_ptype = tunnels_info[i].ptype;
1027                 } else {
1028                         tunnel_ptype = 0;
1029                         break;
1030                 }
1031         }
1032         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1033 }
1034
1035 /**
1036  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1037  * flow.
1038  *
1039  * @param[in] dev
1040  *   Pointer to the Ethernet device structure.
1041  * @param[in] dev_handle
1042  *   Pointer to device flow handle structure.
1043  */
1044 void
1045 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1046                        struct mlx5_flow_handle *dev_handle)
1047 {
1048         struct mlx5_priv *priv = dev->data->dev_private;
1049         const int mark = dev_handle->mark;
1050         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1051         struct mlx5_ind_table_obj *ind_tbl = NULL;
1052         unsigned int i;
1053
1054         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1055                 struct mlx5_hrxq *hrxq;
1056
1057                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1058                               dev_handle->rix_hrxq);
1059                 if (hrxq)
1060                         ind_tbl = hrxq->ind_table;
1061         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1062                 struct mlx5_shared_action_rss *shared_rss;
1063
1064                 shared_rss = mlx5_ipool_get
1065                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1066                          dev_handle->rix_srss);
1067                 if (shared_rss)
1068                         ind_tbl = shared_rss->ind_tbl;
1069         }
1070         if (!ind_tbl)
1071                 return;
1072         for (i = 0; i != ind_tbl->queues_n; ++i) {
1073                 int idx = ind_tbl->queues[i];
1074                 struct mlx5_rxq_ctrl *rxq_ctrl =
1075                         container_of((*priv->rxqs)[idx],
1076                                      struct mlx5_rxq_ctrl, rxq);
1077
1078                 /*
1079                  * To support metadata register copy on Tx loopback,
1080                  * this must be always enabled (metadata may arive
1081                  * from other port - not from local flows only.
1082                  */
1083                 if (priv->config.dv_flow_en &&
1084                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1085                     mlx5_flow_ext_mreg_supported(dev)) {
1086                         rxq_ctrl->rxq.mark = 1;
1087                         rxq_ctrl->flow_mark_n = 1;
1088                 } else if (mark) {
1089                         rxq_ctrl->rxq.mark = 1;
1090                         rxq_ctrl->flow_mark_n++;
1091                 }
1092                 if (tunnel) {
1093                         unsigned int j;
1094
1095                         /* Increase the counter matching the flow. */
1096                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1097                                 if ((tunnels_info[j].tunnel &
1098                                      dev_handle->layers) ==
1099                                     tunnels_info[j].tunnel) {
1100                                         rxq_ctrl->flow_tunnels_n[j]++;
1101                                         break;
1102                                 }
1103                         }
1104                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1105                 }
1106         }
1107 }
1108
1109 /**
1110  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1111  *
1112  * @param[in] dev
1113  *   Pointer to the Ethernet device structure.
1114  * @param[in] flow
1115  *   Pointer to flow structure.
1116  */
1117 static void
1118 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1119 {
1120         struct mlx5_priv *priv = dev->data->dev_private;
1121         uint32_t handle_idx;
1122         struct mlx5_flow_handle *dev_handle;
1123
1124         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1125                        handle_idx, dev_handle, next)
1126                 flow_drv_rxq_flags_set(dev, dev_handle);
1127 }
1128
1129 /**
1130  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1131  * device flow if no other flow uses it with the same kind of request.
1132  *
1133  * @param dev
1134  *   Pointer to Ethernet device.
1135  * @param[in] dev_handle
1136  *   Pointer to the device flow handle structure.
1137  */
1138 static void
1139 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1140                         struct mlx5_flow_handle *dev_handle)
1141 {
1142         struct mlx5_priv *priv = dev->data->dev_private;
1143         const int mark = dev_handle->mark;
1144         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1145         struct mlx5_ind_table_obj *ind_tbl = NULL;
1146         unsigned int i;
1147
1148         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1149                 struct mlx5_hrxq *hrxq;
1150
1151                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1152                               dev_handle->rix_hrxq);
1153                 if (hrxq)
1154                         ind_tbl = hrxq->ind_table;
1155         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1156                 struct mlx5_shared_action_rss *shared_rss;
1157
1158                 shared_rss = mlx5_ipool_get
1159                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1160                          dev_handle->rix_srss);
1161                 if (shared_rss)
1162                         ind_tbl = shared_rss->ind_tbl;
1163         }
1164         if (!ind_tbl)
1165                 return;
1166         MLX5_ASSERT(dev->data->dev_started);
1167         for (i = 0; i != ind_tbl->queues_n; ++i) {
1168                 int idx = ind_tbl->queues[i];
1169                 struct mlx5_rxq_ctrl *rxq_ctrl =
1170                         container_of((*priv->rxqs)[idx],
1171                                      struct mlx5_rxq_ctrl, rxq);
1172
1173                 if (priv->config.dv_flow_en &&
1174                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1175                     mlx5_flow_ext_mreg_supported(dev)) {
1176                         rxq_ctrl->rxq.mark = 1;
1177                         rxq_ctrl->flow_mark_n = 1;
1178                 } else if (mark) {
1179                         rxq_ctrl->flow_mark_n--;
1180                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1181                 }
1182                 if (tunnel) {
1183                         unsigned int j;
1184
1185                         /* Decrease the counter matching the flow. */
1186                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1187                                 if ((tunnels_info[j].tunnel &
1188                                      dev_handle->layers) ==
1189                                     tunnels_info[j].tunnel) {
1190                                         rxq_ctrl->flow_tunnels_n[j]--;
1191                                         break;
1192                                 }
1193                         }
1194                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1195                 }
1196         }
1197 }
1198
1199 /**
1200  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1201  * @p flow if no other flow uses it with the same kind of request.
1202  *
1203  * @param dev
1204  *   Pointer to Ethernet device.
1205  * @param[in] flow
1206  *   Pointer to the flow.
1207  */
1208 static void
1209 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1210 {
1211         struct mlx5_priv *priv = dev->data->dev_private;
1212         uint32_t handle_idx;
1213         struct mlx5_flow_handle *dev_handle;
1214
1215         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1216                        handle_idx, dev_handle, next)
1217                 flow_drv_rxq_flags_trim(dev, dev_handle);
1218 }
1219
1220 /**
1221  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1222  *
1223  * @param dev
1224  *   Pointer to Ethernet device.
1225  */
1226 static void
1227 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1228 {
1229         struct mlx5_priv *priv = dev->data->dev_private;
1230         unsigned int i;
1231
1232         for (i = 0; i != priv->rxqs_n; ++i) {
1233                 struct mlx5_rxq_ctrl *rxq_ctrl;
1234                 unsigned int j;
1235
1236                 if (!(*priv->rxqs)[i])
1237                         continue;
1238                 rxq_ctrl = container_of((*priv->rxqs)[i],
1239                                         struct mlx5_rxq_ctrl, rxq);
1240                 rxq_ctrl->flow_mark_n = 0;
1241                 rxq_ctrl->rxq.mark = 0;
1242                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1243                         rxq_ctrl->flow_tunnels_n[j] = 0;
1244                 rxq_ctrl->rxq.tunnel = 0;
1245         }
1246 }
1247
1248 /**
1249  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1250  *
1251  * @param[in] dev
1252  *   Pointer to the Ethernet device structure.
1253  */
1254 void
1255 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1256 {
1257         struct mlx5_priv *priv = dev->data->dev_private;
1258         struct mlx5_rxq_data *data;
1259         unsigned int i;
1260
1261         for (i = 0; i != priv->rxqs_n; ++i) {
1262                 if (!(*priv->rxqs)[i])
1263                         continue;
1264                 data = (*priv->rxqs)[i];
1265                 if (!rte_flow_dynf_metadata_avail()) {
1266                         data->dynf_meta = 0;
1267                         data->flow_meta_mask = 0;
1268                         data->flow_meta_offset = -1;
1269                         data->flow_meta_port_mask = 0;
1270                 } else {
1271                         data->dynf_meta = 1;
1272                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1273                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1274                         data->flow_meta_port_mask = (uint32_t)~0;
1275                         if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16)
1276                                 data->flow_meta_port_mask >>= 16;
1277                 }
1278         }
1279 }
1280
1281 /*
1282  * return a pointer to the desired action in the list of actions.
1283  *
1284  * @param[in] actions
1285  *   The list of actions to search the action in.
1286  * @param[in] action
1287  *   The action to find.
1288  *
1289  * @return
1290  *   Pointer to the action in the list, if found. NULL otherwise.
1291  */
1292 const struct rte_flow_action *
1293 mlx5_flow_find_action(const struct rte_flow_action *actions,
1294                       enum rte_flow_action_type action)
1295 {
1296         if (actions == NULL)
1297                 return NULL;
1298         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1299                 if (actions->type == action)
1300                         return actions;
1301         return NULL;
1302 }
1303
1304 /*
1305  * Validate the flag action.
1306  *
1307  * @param[in] action_flags
1308  *   Bit-fields that holds the actions detected until now.
1309  * @param[in] attr
1310  *   Attributes of flow that includes this action.
1311  * @param[out] error
1312  *   Pointer to error structure.
1313  *
1314  * @return
1315  *   0 on success, a negative errno value otherwise and rte_errno is set.
1316  */
1317 int
1318 mlx5_flow_validate_action_flag(uint64_t action_flags,
1319                                const struct rte_flow_attr *attr,
1320                                struct rte_flow_error *error)
1321 {
1322         if (action_flags & MLX5_FLOW_ACTION_MARK)
1323                 return rte_flow_error_set(error, EINVAL,
1324                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1325                                           "can't mark and flag in same flow");
1326         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1327                 return rte_flow_error_set(error, EINVAL,
1328                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1329                                           "can't have 2 flag"
1330                                           " actions in same flow");
1331         if (attr->egress)
1332                 return rte_flow_error_set(error, ENOTSUP,
1333                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1334                                           "flag action not supported for "
1335                                           "egress");
1336         return 0;
1337 }
1338
1339 /*
1340  * Validate the mark action.
1341  *
1342  * @param[in] action
1343  *   Pointer to the queue action.
1344  * @param[in] action_flags
1345  *   Bit-fields that holds the actions detected until now.
1346  * @param[in] attr
1347  *   Attributes of flow that includes this action.
1348  * @param[out] error
1349  *   Pointer to error structure.
1350  *
1351  * @return
1352  *   0 on success, a negative errno value otherwise and rte_errno is set.
1353  */
1354 int
1355 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1356                                uint64_t action_flags,
1357                                const struct rte_flow_attr *attr,
1358                                struct rte_flow_error *error)
1359 {
1360         const struct rte_flow_action_mark *mark = action->conf;
1361
1362         if (!mark)
1363                 return rte_flow_error_set(error, EINVAL,
1364                                           RTE_FLOW_ERROR_TYPE_ACTION,
1365                                           action,
1366                                           "configuration cannot be null");
1367         if (mark->id >= MLX5_FLOW_MARK_MAX)
1368                 return rte_flow_error_set(error, EINVAL,
1369                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1370                                           &mark->id,
1371                                           "mark id must in 0 <= id < "
1372                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1373         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1374                 return rte_flow_error_set(error, EINVAL,
1375                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1376                                           "can't flag and mark in same flow");
1377         if (action_flags & MLX5_FLOW_ACTION_MARK)
1378                 return rte_flow_error_set(error, EINVAL,
1379                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1380                                           "can't have 2 mark actions in same"
1381                                           " flow");
1382         if (attr->egress)
1383                 return rte_flow_error_set(error, ENOTSUP,
1384                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1385                                           "mark action not supported for "
1386                                           "egress");
1387         return 0;
1388 }
1389
1390 /*
1391  * Validate the drop action.
1392  *
1393  * @param[in] action_flags
1394  *   Bit-fields that holds the actions detected until now.
1395  * @param[in] attr
1396  *   Attributes of flow that includes this action.
1397  * @param[out] error
1398  *   Pointer to error structure.
1399  *
1400  * @return
1401  *   0 on success, a negative errno value otherwise and rte_errno is set.
1402  */
1403 int
1404 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1405                                const struct rte_flow_attr *attr,
1406                                struct rte_flow_error *error)
1407 {
1408         if (attr->egress)
1409                 return rte_flow_error_set(error, ENOTSUP,
1410                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1411                                           "drop action not supported for "
1412                                           "egress");
1413         return 0;
1414 }
1415
1416 /*
1417  * Validate the queue action.
1418  *
1419  * @param[in] action
1420  *   Pointer to the queue action.
1421  * @param[in] action_flags
1422  *   Bit-fields that holds the actions detected until now.
1423  * @param[in] dev
1424  *   Pointer to the Ethernet device structure.
1425  * @param[in] attr
1426  *   Attributes of flow that includes this action.
1427  * @param[out] error
1428  *   Pointer to error structure.
1429  *
1430  * @return
1431  *   0 on success, a negative errno value otherwise and rte_errno is set.
1432  */
1433 int
1434 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1435                                 uint64_t action_flags,
1436                                 struct rte_eth_dev *dev,
1437                                 const struct rte_flow_attr *attr,
1438                                 struct rte_flow_error *error)
1439 {
1440         struct mlx5_priv *priv = dev->data->dev_private;
1441         const struct rte_flow_action_queue *queue = action->conf;
1442
1443         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1444                 return rte_flow_error_set(error, EINVAL,
1445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1446                                           "can't have 2 fate actions in"
1447                                           " same flow");
1448         if (!priv->rxqs_n)
1449                 return rte_flow_error_set(error, EINVAL,
1450                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1451                                           NULL, "No Rx queues configured");
1452         if (queue->index >= priv->rxqs_n)
1453                 return rte_flow_error_set(error, EINVAL,
1454                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1455                                           &queue->index,
1456                                           "queue index out of range");
1457         if (!(*priv->rxqs)[queue->index])
1458                 return rte_flow_error_set(error, EINVAL,
1459                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1460                                           &queue->index,
1461                                           "queue is not configured");
1462         if (attr->egress)
1463                 return rte_flow_error_set(error, ENOTSUP,
1464                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1465                                           "queue action not supported for "
1466                                           "egress");
1467         return 0;
1468 }
1469
1470 /*
1471  * Validate the rss action.
1472  *
1473  * @param[in] dev
1474  *   Pointer to the Ethernet device structure.
1475  * @param[in] action
1476  *   Pointer to the queue action.
1477  * @param[out] error
1478  *   Pointer to error structure.
1479  *
1480  * @return
1481  *   0 on success, a negative errno value otherwise and rte_errno is set.
1482  */
1483 int
1484 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1485                          const struct rte_flow_action *action,
1486                          struct rte_flow_error *error)
1487 {
1488         struct mlx5_priv *priv = dev->data->dev_private;
1489         const struct rte_flow_action_rss *rss = action->conf;
1490         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1491         unsigned int i;
1492
1493         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1494             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1495                 return rte_flow_error_set(error, ENOTSUP,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1497                                           &rss->func,
1498                                           "RSS hash function not supported");
1499 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1500         if (rss->level > 2)
1501 #else
1502         if (rss->level > 1)
1503 #endif
1504                 return rte_flow_error_set(error, ENOTSUP,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1506                                           &rss->level,
1507                                           "tunnel RSS is not supported");
1508         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1509         if (rss->key_len == 0 && rss->key != NULL)
1510                 return rte_flow_error_set(error, ENOTSUP,
1511                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1512                                           &rss->key_len,
1513                                           "RSS hash key length 0");
1514         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1515                 return rte_flow_error_set(error, ENOTSUP,
1516                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1517                                           &rss->key_len,
1518                                           "RSS hash key too small");
1519         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1520                 return rte_flow_error_set(error, ENOTSUP,
1521                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1522                                           &rss->key_len,
1523                                           "RSS hash key too large");
1524         if (rss->queue_num > priv->config.ind_table_max_size)
1525                 return rte_flow_error_set(error, ENOTSUP,
1526                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1527                                           &rss->queue_num,
1528                                           "number of queues too large");
1529         if (rss->types & MLX5_RSS_HF_MASK)
1530                 return rte_flow_error_set(error, ENOTSUP,
1531                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1532                                           &rss->types,
1533                                           "some RSS protocols are not"
1534                                           " supported");
1535         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1536             !(rss->types & ETH_RSS_IP))
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1539                                           "L3 partial RSS requested but L3 RSS"
1540                                           " type not specified");
1541         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1542             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1543                 return rte_flow_error_set(error, EINVAL,
1544                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1545                                           "L4 partial RSS requested but L4 RSS"
1546                                           " type not specified");
1547         if (!priv->rxqs_n)
1548                 return rte_flow_error_set(error, EINVAL,
1549                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1550                                           NULL, "No Rx queues configured");
1551         if (!rss->queue_num)
1552                 return rte_flow_error_set(error, EINVAL,
1553                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1554                                           NULL, "No queues configured");
1555         for (i = 0; i != rss->queue_num; ++i) {
1556                 struct mlx5_rxq_ctrl *rxq_ctrl;
1557
1558                 if (rss->queue[i] >= priv->rxqs_n)
1559                         return rte_flow_error_set
1560                                 (error, EINVAL,
1561                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1562                                  &rss->queue[i], "queue index out of range");
1563                 if (!(*priv->rxqs)[rss->queue[i]])
1564                         return rte_flow_error_set
1565                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1566                                  &rss->queue[i], "queue is not configured");
1567                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1568                                         struct mlx5_rxq_ctrl, rxq);
1569                 if (i == 0)
1570                         rxq_type = rxq_ctrl->type;
1571                 if (rxq_type != rxq_ctrl->type)
1572                         return rte_flow_error_set
1573                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1574                                  &rss->queue[i],
1575                                  "combining hairpin and regular RSS queues is not supported");
1576         }
1577         return 0;
1578 }
1579
1580 /*
1581  * Validate the rss action.
1582  *
1583  * @param[in] action
1584  *   Pointer to the queue action.
1585  * @param[in] action_flags
1586  *   Bit-fields that holds the actions detected until now.
1587  * @param[in] dev
1588  *   Pointer to the Ethernet device structure.
1589  * @param[in] attr
1590  *   Attributes of flow that includes this action.
1591  * @param[in] item_flags
1592  *   Items that were detected.
1593  * @param[out] error
1594  *   Pointer to error structure.
1595  *
1596  * @return
1597  *   0 on success, a negative errno value otherwise and rte_errno is set.
1598  */
1599 int
1600 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1601                               uint64_t action_flags,
1602                               struct rte_eth_dev *dev,
1603                               const struct rte_flow_attr *attr,
1604                               uint64_t item_flags,
1605                               struct rte_flow_error *error)
1606 {
1607         const struct rte_flow_action_rss *rss = action->conf;
1608         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1609         int ret;
1610
1611         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1612                 return rte_flow_error_set(error, EINVAL,
1613                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1614                                           "can't have 2 fate actions"
1615                                           " in same flow");
1616         ret = mlx5_validate_action_rss(dev, action, error);
1617         if (ret)
1618                 return ret;
1619         if (attr->egress)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1622                                           "rss action not supported for "
1623                                           "egress");
1624         if (rss->level > 1 && !tunnel)
1625                 return rte_flow_error_set(error, EINVAL,
1626                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1627                                           "inner RSS is not supported for "
1628                                           "non-tunnel flows");
1629         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1630             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1631                 return rte_flow_error_set(error, EINVAL,
1632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1633                                           "RSS on eCPRI is not supported now");
1634         }
1635         return 0;
1636 }
1637
1638 /*
1639  * Validate the default miss action.
1640  *
1641  * @param[in] action_flags
1642  *   Bit-fields that holds the actions detected until now.
1643  * @param[out] error
1644  *   Pointer to error structure.
1645  *
1646  * @return
1647  *   0 on success, a negative errno value otherwise and rte_errno is set.
1648  */
1649 int
1650 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1651                                 const struct rte_flow_attr *attr,
1652                                 struct rte_flow_error *error)
1653 {
1654         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1655                 return rte_flow_error_set(error, EINVAL,
1656                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1657                                           "can't have 2 fate actions in"
1658                                           " same flow");
1659         if (attr->egress)
1660                 return rte_flow_error_set(error, ENOTSUP,
1661                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1662                                           "default miss action not supported "
1663                                           "for egress");
1664         if (attr->group)
1665                 return rte_flow_error_set(error, ENOTSUP,
1666                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1667                                           "only group 0 is supported");
1668         if (attr->transfer)
1669                 return rte_flow_error_set(error, ENOTSUP,
1670                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1671                                           NULL, "transfer is not supported");
1672         return 0;
1673 }
1674
1675 /*
1676  * Validate the count action.
1677  *
1678  * @param[in] dev
1679  *   Pointer to the Ethernet device structure.
1680  * @param[in] attr
1681  *   Attributes of flow that includes this action.
1682  * @param[out] error
1683  *   Pointer to error structure.
1684  *
1685  * @return
1686  *   0 on success, a negative errno value otherwise and rte_errno is set.
1687  */
1688 int
1689 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1690                                 const struct rte_flow_attr *attr,
1691                                 struct rte_flow_error *error)
1692 {
1693         if (attr->egress)
1694                 return rte_flow_error_set(error, ENOTSUP,
1695                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1696                                           "count action not supported for "
1697                                           "egress");
1698         return 0;
1699 }
1700
1701 /*
1702  * Validate the ASO CT action.
1703  *
1704  * @param[in] dev
1705  *   Pointer to the Ethernet device structure.
1706  * @param[in] conntrack
1707  *   Pointer to the CT action profile.
1708  * @param[out] error
1709  *   Pointer to error structure.
1710  *
1711  * @return
1712  *   0 on success, a negative errno value otherwise and rte_errno is set.
1713  */
1714 int
1715 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1716                         const struct rte_flow_action_conntrack *conntrack,
1717                         struct rte_flow_error *error)
1718 {
1719         RTE_SET_USED(dev);
1720
1721         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1722                 return rte_flow_error_set(error, EINVAL,
1723                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1724                                           "Invalid CT state");
1725         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1726                 return rte_flow_error_set(error, EINVAL,
1727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1728                                           "Invalid last TCP packet flag");
1729         return 0;
1730 }
1731
1732 /**
1733  * Verify the @p attributes will be correctly understood by the NIC and store
1734  * them in the @p flow if everything is correct.
1735  *
1736  * @param[in] dev
1737  *   Pointer to the Ethernet device structure.
1738  * @param[in] attributes
1739  *   Pointer to flow attributes
1740  * @param[out] error
1741  *   Pointer to error structure.
1742  *
1743  * @return
1744  *   0 on success, a negative errno value otherwise and rte_errno is set.
1745  */
1746 int
1747 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1748                               const struct rte_flow_attr *attributes,
1749                               struct rte_flow_error *error)
1750 {
1751         struct mlx5_priv *priv = dev->data->dev_private;
1752         uint32_t priority_max = priv->config.flow_prio - 1;
1753
1754         if (attributes->group)
1755                 return rte_flow_error_set(error, ENOTSUP,
1756                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1757                                           NULL, "groups is not supported");
1758         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1759             attributes->priority >= priority_max)
1760                 return rte_flow_error_set(error, ENOTSUP,
1761                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1762                                           NULL, "priority out of range");
1763         if (attributes->egress)
1764                 return rte_flow_error_set(error, ENOTSUP,
1765                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1766                                           "egress is not supported");
1767         if (attributes->transfer && !priv->config.dv_esw_en)
1768                 return rte_flow_error_set(error, ENOTSUP,
1769                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1770                                           NULL, "transfer is not supported");
1771         if (!attributes->ingress)
1772                 return rte_flow_error_set(error, EINVAL,
1773                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1774                                           NULL,
1775                                           "ingress attribute is mandatory");
1776         return 0;
1777 }
1778
1779 /**
1780  * Validate ICMP6 item.
1781  *
1782  * @param[in] item
1783  *   Item specification.
1784  * @param[in] item_flags
1785  *   Bit-fields that holds the items detected until now.
1786  * @param[in] ext_vlan_sup
1787  *   Whether extended VLAN features are supported or not.
1788  * @param[out] error
1789  *   Pointer to error structure.
1790  *
1791  * @return
1792  *   0 on success, a negative errno value otherwise and rte_errno is set.
1793  */
1794 int
1795 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1796                                uint64_t item_flags,
1797                                uint8_t target_protocol,
1798                                struct rte_flow_error *error)
1799 {
1800         const struct rte_flow_item_icmp6 *mask = item->mask;
1801         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1802         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1803                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1804         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1805                                       MLX5_FLOW_LAYER_OUTER_L4;
1806         int ret;
1807
1808         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1809                 return rte_flow_error_set(error, EINVAL,
1810                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1811                                           "protocol filtering not compatible"
1812                                           " with ICMP6 layer");
1813         if (!(item_flags & l3m))
1814                 return rte_flow_error_set(error, EINVAL,
1815                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1816                                           "IPv6 is mandatory to filter on"
1817                                           " ICMP6");
1818         if (item_flags & l4m)
1819                 return rte_flow_error_set(error, EINVAL,
1820                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1821                                           "multiple L4 layers not supported");
1822         if (!mask)
1823                 mask = &rte_flow_item_icmp6_mask;
1824         ret = mlx5_flow_item_acceptable
1825                 (item, (const uint8_t *)mask,
1826                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1827                  sizeof(struct rte_flow_item_icmp6),
1828                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1829         if (ret < 0)
1830                 return ret;
1831         return 0;
1832 }
1833
1834 /**
1835  * Validate ICMP item.
1836  *
1837  * @param[in] item
1838  *   Item specification.
1839  * @param[in] item_flags
1840  *   Bit-fields that holds the items detected until now.
1841  * @param[out] error
1842  *   Pointer to error structure.
1843  *
1844  * @return
1845  *   0 on success, a negative errno value otherwise and rte_errno is set.
1846  */
1847 int
1848 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1849                              uint64_t item_flags,
1850                              uint8_t target_protocol,
1851                              struct rte_flow_error *error)
1852 {
1853         const struct rte_flow_item_icmp *mask = item->mask;
1854         const struct rte_flow_item_icmp nic_mask = {
1855                 .hdr.icmp_type = 0xff,
1856                 .hdr.icmp_code = 0xff,
1857                 .hdr.icmp_ident = RTE_BE16(0xffff),
1858                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1859         };
1860         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1861         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1862                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1863         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1864                                       MLX5_FLOW_LAYER_OUTER_L4;
1865         int ret;
1866
1867         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1868                 return rte_flow_error_set(error, EINVAL,
1869                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1870                                           "protocol filtering not compatible"
1871                                           " with ICMP layer");
1872         if (!(item_flags & l3m))
1873                 return rte_flow_error_set(error, EINVAL,
1874                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1875                                           "IPv4 is mandatory to filter"
1876                                           " on ICMP");
1877         if (item_flags & l4m)
1878                 return rte_flow_error_set(error, EINVAL,
1879                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1880                                           "multiple L4 layers not supported");
1881         if (!mask)
1882                 mask = &nic_mask;
1883         ret = mlx5_flow_item_acceptable
1884                 (item, (const uint8_t *)mask,
1885                  (const uint8_t *)&nic_mask,
1886                  sizeof(struct rte_flow_item_icmp),
1887                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1888         if (ret < 0)
1889                 return ret;
1890         return 0;
1891 }
1892
1893 /**
1894  * Validate Ethernet item.
1895  *
1896  * @param[in] item
1897  *   Item specification.
1898  * @param[in] item_flags
1899  *   Bit-fields that holds the items detected until now.
1900  * @param[out] error
1901  *   Pointer to error structure.
1902  *
1903  * @return
1904  *   0 on success, a negative errno value otherwise and rte_errno is set.
1905  */
1906 int
1907 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1908                             uint64_t item_flags, bool ext_vlan_sup,
1909                             struct rte_flow_error *error)
1910 {
1911         const struct rte_flow_item_eth *mask = item->mask;
1912         const struct rte_flow_item_eth nic_mask = {
1913                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1914                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1915                 .type = RTE_BE16(0xffff),
1916                 .has_vlan = ext_vlan_sup ? 1 : 0,
1917         };
1918         int ret;
1919         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1920         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1921                                        MLX5_FLOW_LAYER_OUTER_L2;
1922
1923         if (item_flags & ethm)
1924                 return rte_flow_error_set(error, ENOTSUP,
1925                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1926                                           "multiple L2 layers not supported");
1927         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1928             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1929                 return rte_flow_error_set(error, EINVAL,
1930                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1931                                           "L2 layer should not follow "
1932                                           "L3 layers");
1933         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1934             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1935                 return rte_flow_error_set(error, EINVAL,
1936                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1937                                           "L2 layer should not follow VLAN");
1938         if (!mask)
1939                 mask = &rte_flow_item_eth_mask;
1940         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1941                                         (const uint8_t *)&nic_mask,
1942                                         sizeof(struct rte_flow_item_eth),
1943                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1944         return ret;
1945 }
1946
1947 /**
1948  * Validate VLAN item.
1949  *
1950  * @param[in] item
1951  *   Item specification.
1952  * @param[in] item_flags
1953  *   Bit-fields that holds the items detected until now.
1954  * @param[in] dev
1955  *   Ethernet device flow is being created on.
1956  * @param[out] error
1957  *   Pointer to error structure.
1958  *
1959  * @return
1960  *   0 on success, a negative errno value otherwise and rte_errno is set.
1961  */
1962 int
1963 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1964                              uint64_t item_flags,
1965                              struct rte_eth_dev *dev,
1966                              struct rte_flow_error *error)
1967 {
1968         const struct rte_flow_item_vlan *spec = item->spec;
1969         const struct rte_flow_item_vlan *mask = item->mask;
1970         const struct rte_flow_item_vlan nic_mask = {
1971                 .tci = RTE_BE16(UINT16_MAX),
1972                 .inner_type = RTE_BE16(UINT16_MAX),
1973         };
1974         uint16_t vlan_tag = 0;
1975         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1976         int ret;
1977         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1978                                         MLX5_FLOW_LAYER_INNER_L4) :
1979                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1980                                         MLX5_FLOW_LAYER_OUTER_L4);
1981         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1982                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1983
1984         if (item_flags & vlanm)
1985                 return rte_flow_error_set(error, EINVAL,
1986                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1987                                           "multiple VLAN layers not supported");
1988         else if ((item_flags & l34m) != 0)
1989                 return rte_flow_error_set(error, EINVAL,
1990                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1991                                           "VLAN cannot follow L3/L4 layer");
1992         if (!mask)
1993                 mask = &rte_flow_item_vlan_mask;
1994         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1995                                         (const uint8_t *)&nic_mask,
1996                                         sizeof(struct rte_flow_item_vlan),
1997                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1998         if (ret)
1999                 return ret;
2000         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2001                 struct mlx5_priv *priv = dev->data->dev_private;
2002
2003                 if (priv->vmwa_context) {
2004                         /*
2005                          * Non-NULL context means we have a virtual machine
2006                          * and SR-IOV enabled, we have to create VLAN interface
2007                          * to make hypervisor to setup E-Switch vport
2008                          * context correctly. We avoid creating the multiple
2009                          * VLAN interfaces, so we cannot support VLAN tag mask.
2010                          */
2011                         return rte_flow_error_set(error, EINVAL,
2012                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2013                                                   item,
2014                                                   "VLAN tag mask is not"
2015                                                   " supported in virtual"
2016                                                   " environment");
2017                 }
2018         }
2019         if (spec) {
2020                 vlan_tag = spec->tci;
2021                 vlan_tag &= mask->tci;
2022         }
2023         /*
2024          * From verbs perspective an empty VLAN is equivalent
2025          * to a packet without VLAN layer.
2026          */
2027         if (!vlan_tag)
2028                 return rte_flow_error_set(error, EINVAL,
2029                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2030                                           item->spec,
2031                                           "VLAN cannot be empty");
2032         return 0;
2033 }
2034
2035 /**
2036  * Validate IPV4 item.
2037  *
2038  * @param[in] item
2039  *   Item specification.
2040  * @param[in] item_flags
2041  *   Bit-fields that holds the items detected until now.
2042  * @param[in] last_item
2043  *   Previous validated item in the pattern items.
2044  * @param[in] ether_type
2045  *   Type in the ethernet layer header (including dot1q).
2046  * @param[in] acc_mask
2047  *   Acceptable mask, if NULL default internal default mask
2048  *   will be used to check whether item fields are supported.
2049  * @param[in] range_accepted
2050  *   True if range of values is accepted for specific fields, false otherwise.
2051  * @param[out] error
2052  *   Pointer to error structure.
2053  *
2054  * @return
2055  *   0 on success, a negative errno value otherwise and rte_errno is set.
2056  */
2057 int
2058 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2059                              uint64_t item_flags,
2060                              uint64_t last_item,
2061                              uint16_t ether_type,
2062                              const struct rte_flow_item_ipv4 *acc_mask,
2063                              bool range_accepted,
2064                              struct rte_flow_error *error)
2065 {
2066         const struct rte_flow_item_ipv4 *mask = item->mask;
2067         const struct rte_flow_item_ipv4 *spec = item->spec;
2068         const struct rte_flow_item_ipv4 nic_mask = {
2069                 .hdr = {
2070                         .src_addr = RTE_BE32(0xffffffff),
2071                         .dst_addr = RTE_BE32(0xffffffff),
2072                         .type_of_service = 0xff,
2073                         .next_proto_id = 0xff,
2074                 },
2075         };
2076         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2077         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2078                                       MLX5_FLOW_LAYER_OUTER_L3;
2079         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2080                                       MLX5_FLOW_LAYER_OUTER_L4;
2081         int ret;
2082         uint8_t next_proto = 0xFF;
2083         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2084                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2085                                   MLX5_FLOW_LAYER_INNER_VLAN);
2086
2087         if ((last_item & l2_vlan) && ether_type &&
2088             ether_type != RTE_ETHER_TYPE_IPV4)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2091                                           "IPv4 cannot follow L2/VLAN layer "
2092                                           "which ether type is not IPv4");
2093         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2094                 if (mask && spec)
2095                         next_proto = mask->hdr.next_proto_id &
2096                                      spec->hdr.next_proto_id;
2097                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2098                         return rte_flow_error_set(error, EINVAL,
2099                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2100                                                   item,
2101                                                   "multiple tunnel "
2102                                                   "not supported");
2103         }
2104         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2105                 return rte_flow_error_set(error, EINVAL,
2106                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2107                                           "wrong tunnel type - IPv6 specified "
2108                                           "but IPv4 item provided");
2109         if (item_flags & l3m)
2110                 return rte_flow_error_set(error, ENOTSUP,
2111                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2112                                           "multiple L3 layers not supported");
2113         else if (item_flags & l4m)
2114                 return rte_flow_error_set(error, EINVAL,
2115                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2116                                           "L3 cannot follow an L4 layer.");
2117         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2118                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2119                 return rte_flow_error_set(error, EINVAL,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2121                                           "L3 cannot follow an NVGRE layer.");
2122         if (!mask)
2123                 mask = &rte_flow_item_ipv4_mask;
2124         else if (mask->hdr.next_proto_id != 0 &&
2125                  mask->hdr.next_proto_id != 0xff)
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2128                                           "partial mask is not supported"
2129                                           " for protocol");
2130         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2131                                         acc_mask ? (const uint8_t *)acc_mask
2132                                                  : (const uint8_t *)&nic_mask,
2133                                         sizeof(struct rte_flow_item_ipv4),
2134                                         range_accepted, error);
2135         if (ret < 0)
2136                 return ret;
2137         return 0;
2138 }
2139
2140 /**
2141  * Validate IPV6 item.
2142  *
2143  * @param[in] item
2144  *   Item specification.
2145  * @param[in] item_flags
2146  *   Bit-fields that holds the items detected until now.
2147  * @param[in] last_item
2148  *   Previous validated item in the pattern items.
2149  * @param[in] ether_type
2150  *   Type in the ethernet layer header (including dot1q).
2151  * @param[in] acc_mask
2152  *   Acceptable mask, if NULL default internal default mask
2153  *   will be used to check whether item fields are supported.
2154  * @param[out] error
2155  *   Pointer to error structure.
2156  *
2157  * @return
2158  *   0 on success, a negative errno value otherwise and rte_errno is set.
2159  */
2160 int
2161 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2162                              uint64_t item_flags,
2163                              uint64_t last_item,
2164                              uint16_t ether_type,
2165                              const struct rte_flow_item_ipv6 *acc_mask,
2166                              struct rte_flow_error *error)
2167 {
2168         const struct rte_flow_item_ipv6 *mask = item->mask;
2169         const struct rte_flow_item_ipv6 *spec = item->spec;
2170         const struct rte_flow_item_ipv6 nic_mask = {
2171                 .hdr = {
2172                         .src_addr =
2173                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2174                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2175                         .dst_addr =
2176                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2177                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2178                         .vtc_flow = RTE_BE32(0xffffffff),
2179                         .proto = 0xff,
2180                 },
2181         };
2182         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2183         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2184                                       MLX5_FLOW_LAYER_OUTER_L3;
2185         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2186                                       MLX5_FLOW_LAYER_OUTER_L4;
2187         int ret;
2188         uint8_t next_proto = 0xFF;
2189         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2190                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2191                                   MLX5_FLOW_LAYER_INNER_VLAN);
2192
2193         if ((last_item & l2_vlan) && ether_type &&
2194             ether_type != RTE_ETHER_TYPE_IPV6)
2195                 return rte_flow_error_set(error, EINVAL,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2197                                           "IPv6 cannot follow L2/VLAN layer "
2198                                           "which ether type is not IPv6");
2199         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2200                 next_proto = spec->hdr.proto;
2201         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2202                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2203                         return rte_flow_error_set(error, EINVAL,
2204                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2205                                                   item,
2206                                                   "multiple tunnel "
2207                                                   "not supported");
2208         }
2209         if (next_proto == IPPROTO_HOPOPTS  ||
2210             next_proto == IPPROTO_ROUTING  ||
2211             next_proto == IPPROTO_FRAGMENT ||
2212             next_proto == IPPROTO_ESP      ||
2213             next_proto == IPPROTO_AH       ||
2214             next_proto == IPPROTO_DSTOPTS)
2215                 return rte_flow_error_set(error, EINVAL,
2216                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2217                                           "IPv6 proto (next header) should "
2218                                           "not be set as extension header");
2219         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2220                 return rte_flow_error_set(error, EINVAL,
2221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2222                                           "wrong tunnel type - IPv4 specified "
2223                                           "but IPv6 item provided");
2224         if (item_flags & l3m)
2225                 return rte_flow_error_set(error, ENOTSUP,
2226                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2227                                           "multiple L3 layers not supported");
2228         else if (item_flags & l4m)
2229                 return rte_flow_error_set(error, EINVAL,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2231                                           "L3 cannot follow an L4 layer.");
2232         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2233                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2234                 return rte_flow_error_set(error, EINVAL,
2235                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2236                                           "L3 cannot follow an NVGRE layer.");
2237         if (!mask)
2238                 mask = &rte_flow_item_ipv6_mask;
2239         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2240                                         acc_mask ? (const uint8_t *)acc_mask
2241                                                  : (const uint8_t *)&nic_mask,
2242                                         sizeof(struct rte_flow_item_ipv6),
2243                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2244         if (ret < 0)
2245                 return ret;
2246         return 0;
2247 }
2248
2249 /**
2250  * Validate UDP item.
2251  *
2252  * @param[in] item
2253  *   Item specification.
2254  * @param[in] item_flags
2255  *   Bit-fields that holds the items detected until now.
2256  * @param[in] target_protocol
2257  *   The next protocol in the previous item.
2258  * @param[in] flow_mask
2259  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2260  * @param[out] error
2261  *   Pointer to error structure.
2262  *
2263  * @return
2264  *   0 on success, a negative errno value otherwise and rte_errno is set.
2265  */
2266 int
2267 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2268                             uint64_t item_flags,
2269                             uint8_t target_protocol,
2270                             struct rte_flow_error *error)
2271 {
2272         const struct rte_flow_item_udp *mask = item->mask;
2273         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2274         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2275                                       MLX5_FLOW_LAYER_OUTER_L3;
2276         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2277                                       MLX5_FLOW_LAYER_OUTER_L4;
2278         int ret;
2279
2280         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2281                 return rte_flow_error_set(error, EINVAL,
2282                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2283                                           "protocol filtering not compatible"
2284                                           " with UDP layer");
2285         if (!(item_flags & l3m))
2286                 return rte_flow_error_set(error, EINVAL,
2287                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2288                                           "L3 is mandatory to filter on L4");
2289         if (item_flags & l4m)
2290                 return rte_flow_error_set(error, EINVAL,
2291                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2292                                           "multiple L4 layers not supported");
2293         if (!mask)
2294                 mask = &rte_flow_item_udp_mask;
2295         ret = mlx5_flow_item_acceptable
2296                 (item, (const uint8_t *)mask,
2297                  (const uint8_t *)&rte_flow_item_udp_mask,
2298                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2299                  error);
2300         if (ret < 0)
2301                 return ret;
2302         return 0;
2303 }
2304
2305 /**
2306  * Validate TCP item.
2307  *
2308  * @param[in] item
2309  *   Item specification.
2310  * @param[in] item_flags
2311  *   Bit-fields that holds the items detected until now.
2312  * @param[in] target_protocol
2313  *   The next protocol in the previous item.
2314  * @param[out] error
2315  *   Pointer to error structure.
2316  *
2317  * @return
2318  *   0 on success, a negative errno value otherwise and rte_errno is set.
2319  */
2320 int
2321 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2322                             uint64_t item_flags,
2323                             uint8_t target_protocol,
2324                             const struct rte_flow_item_tcp *flow_mask,
2325                             struct rte_flow_error *error)
2326 {
2327         const struct rte_flow_item_tcp *mask = item->mask;
2328         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2329         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2330                                       MLX5_FLOW_LAYER_OUTER_L3;
2331         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2332                                       MLX5_FLOW_LAYER_OUTER_L4;
2333         int ret;
2334
2335         MLX5_ASSERT(flow_mask);
2336         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2337                 return rte_flow_error_set(error, EINVAL,
2338                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2339                                           "protocol filtering not compatible"
2340                                           " with TCP layer");
2341         if (!(item_flags & l3m))
2342                 return rte_flow_error_set(error, EINVAL,
2343                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2344                                           "L3 is mandatory to filter on L4");
2345         if (item_flags & l4m)
2346                 return rte_flow_error_set(error, EINVAL,
2347                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2348                                           "multiple L4 layers not supported");
2349         if (!mask)
2350                 mask = &rte_flow_item_tcp_mask;
2351         ret = mlx5_flow_item_acceptable
2352                 (item, (const uint8_t *)mask,
2353                  (const uint8_t *)flow_mask,
2354                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2355                  error);
2356         if (ret < 0)
2357                 return ret;
2358         return 0;
2359 }
2360
2361 /**
2362  * Validate VXLAN item.
2363  *
2364  * @param[in] item
2365  *   Item specification.
2366  * @param[in] item_flags
2367  *   Bit-fields that holds the items detected until now.
2368  * @param[in] target_protocol
2369  *   The next protocol in the previous item.
2370  * @param[out] error
2371  *   Pointer to error structure.
2372  *
2373  * @return
2374  *   0 on success, a negative errno value otherwise and rte_errno is set.
2375  */
2376 int
2377 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2378                               uint64_t item_flags,
2379                               struct rte_flow_error *error)
2380 {
2381         const struct rte_flow_item_vxlan *spec = item->spec;
2382         const struct rte_flow_item_vxlan *mask = item->mask;
2383         int ret;
2384         union vni {
2385                 uint32_t vlan_id;
2386                 uint8_t vni[4];
2387         } id = { .vlan_id = 0, };
2388
2389
2390         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2391                 return rte_flow_error_set(error, ENOTSUP,
2392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2393                                           "multiple tunnel layers not"
2394                                           " supported");
2395         /*
2396          * Verify only UDPv4 is present as defined in
2397          * https://tools.ietf.org/html/rfc7348
2398          */
2399         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2400                 return rte_flow_error_set(error, EINVAL,
2401                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2402                                           "no outer UDP layer found");
2403         if (!mask)
2404                 mask = &rte_flow_item_vxlan_mask;
2405         ret = mlx5_flow_item_acceptable
2406                 (item, (const uint8_t *)mask,
2407                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2408                  sizeof(struct rte_flow_item_vxlan),
2409                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2410         if (ret < 0)
2411                 return ret;
2412         if (spec) {
2413                 memcpy(&id.vni[1], spec->vni, 3);
2414                 memcpy(&id.vni[1], mask->vni, 3);
2415         }
2416         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2419                                           "VXLAN tunnel must be fully defined");
2420         return 0;
2421 }
2422
2423 /**
2424  * Validate VXLAN_GPE item.
2425  *
2426  * @param[in] item
2427  *   Item specification.
2428  * @param[in] item_flags
2429  *   Bit-fields that holds the items detected until now.
2430  * @param[in] priv
2431  *   Pointer to the private data structure.
2432  * @param[in] target_protocol
2433  *   The next protocol in the previous item.
2434  * @param[out] error
2435  *   Pointer to error structure.
2436  *
2437  * @return
2438  *   0 on success, a negative errno value otherwise and rte_errno is set.
2439  */
2440 int
2441 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2442                                   uint64_t item_flags,
2443                                   struct rte_eth_dev *dev,
2444                                   struct rte_flow_error *error)
2445 {
2446         struct mlx5_priv *priv = dev->data->dev_private;
2447         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2448         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2449         int ret;
2450         union vni {
2451                 uint32_t vlan_id;
2452                 uint8_t vni[4];
2453         } id = { .vlan_id = 0, };
2454
2455         if (!priv->config.l3_vxlan_en)
2456                 return rte_flow_error_set(error, ENOTSUP,
2457                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2458                                           "L3 VXLAN is not enabled by device"
2459                                           " parameter and/or not configured in"
2460                                           " firmware");
2461         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2462                 return rte_flow_error_set(error, ENOTSUP,
2463                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2464                                           "multiple tunnel layers not"
2465                                           " supported");
2466         /*
2467          * Verify only UDPv4 is present as defined in
2468          * https://tools.ietf.org/html/rfc7348
2469          */
2470         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2471                 return rte_flow_error_set(error, EINVAL,
2472                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2473                                           "no outer UDP layer found");
2474         if (!mask)
2475                 mask = &rte_flow_item_vxlan_gpe_mask;
2476         ret = mlx5_flow_item_acceptable
2477                 (item, (const uint8_t *)mask,
2478                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2479                  sizeof(struct rte_flow_item_vxlan_gpe),
2480                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2481         if (ret < 0)
2482                 return ret;
2483         if (spec) {
2484                 if (spec->protocol)
2485                         return rte_flow_error_set(error, ENOTSUP,
2486                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2487                                                   item,
2488                                                   "VxLAN-GPE protocol"
2489                                                   " not supported");
2490                 memcpy(&id.vni[1], spec->vni, 3);
2491                 memcpy(&id.vni[1], mask->vni, 3);
2492         }
2493         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2494                 return rte_flow_error_set(error, ENOTSUP,
2495                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2496                                           "VXLAN-GPE tunnel must be fully"
2497                                           " defined");
2498         return 0;
2499 }
2500 /**
2501  * Validate GRE Key item.
2502  *
2503  * @param[in] item
2504  *   Item specification.
2505  * @param[in] item_flags
2506  *   Bit flags to mark detected items.
2507  * @param[in] gre_item
2508  *   Pointer to gre_item
2509  * @param[out] error
2510  *   Pointer to error structure.
2511  *
2512  * @return
2513  *   0 on success, a negative errno value otherwise and rte_errno is set.
2514  */
2515 int
2516 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2517                                 uint64_t item_flags,
2518                                 const struct rte_flow_item *gre_item,
2519                                 struct rte_flow_error *error)
2520 {
2521         const rte_be32_t *mask = item->mask;
2522         int ret = 0;
2523         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2524         const struct rte_flow_item_gre *gre_spec;
2525         const struct rte_flow_item_gre *gre_mask;
2526
2527         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2528                 return rte_flow_error_set(error, ENOTSUP,
2529                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2530                                           "Multiple GRE key not support");
2531         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2532                 return rte_flow_error_set(error, ENOTSUP,
2533                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2534                                           "No preceding GRE header");
2535         if (item_flags & MLX5_FLOW_LAYER_INNER)
2536                 return rte_flow_error_set(error, ENOTSUP,
2537                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2538                                           "GRE key following a wrong item");
2539         gre_mask = gre_item->mask;
2540         if (!gre_mask)
2541                 gre_mask = &rte_flow_item_gre_mask;
2542         gre_spec = gre_item->spec;
2543         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2544                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2545                 return rte_flow_error_set(error, EINVAL,
2546                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2547                                           "Key bit must be on");
2548
2549         if (!mask)
2550                 mask = &gre_key_default_mask;
2551         ret = mlx5_flow_item_acceptable
2552                 (item, (const uint8_t *)mask,
2553                  (const uint8_t *)&gre_key_default_mask,
2554                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2555         return ret;
2556 }
2557
2558 /**
2559  * Validate GRE item.
2560  *
2561  * @param[in] item
2562  *   Item specification.
2563  * @param[in] item_flags
2564  *   Bit flags to mark detected items.
2565  * @param[in] target_protocol
2566  *   The next protocol in the previous item.
2567  * @param[out] error
2568  *   Pointer to error structure.
2569  *
2570  * @return
2571  *   0 on success, a negative errno value otherwise and rte_errno is set.
2572  */
2573 int
2574 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2575                             uint64_t item_flags,
2576                             uint8_t target_protocol,
2577                             struct rte_flow_error *error)
2578 {
2579         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2580         const struct rte_flow_item_gre *mask = item->mask;
2581         int ret;
2582         const struct rte_flow_item_gre nic_mask = {
2583                 .c_rsvd0_ver = RTE_BE16(0xB000),
2584                 .protocol = RTE_BE16(UINT16_MAX),
2585         };
2586
2587         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2588                 return rte_flow_error_set(error, EINVAL,
2589                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2590                                           "protocol filtering not compatible"
2591                                           " with this GRE layer");
2592         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2593                 return rte_flow_error_set(error, ENOTSUP,
2594                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2595                                           "multiple tunnel layers not"
2596                                           " supported");
2597         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2598                 return rte_flow_error_set(error, ENOTSUP,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2600                                           "L3 Layer is missing");
2601         if (!mask)
2602                 mask = &rte_flow_item_gre_mask;
2603         ret = mlx5_flow_item_acceptable
2604                 (item, (const uint8_t *)mask,
2605                  (const uint8_t *)&nic_mask,
2606                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2607                  error);
2608         if (ret < 0)
2609                 return ret;
2610 #ifndef HAVE_MLX5DV_DR
2611 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2612         if (spec && (spec->protocol & mask->protocol))
2613                 return rte_flow_error_set(error, ENOTSUP,
2614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2615                                           "without MPLS support the"
2616                                           " specification cannot be used for"
2617                                           " filtering");
2618 #endif
2619 #endif
2620         return 0;
2621 }
2622
2623 /**
2624  * Validate Geneve item.
2625  *
2626  * @param[in] item
2627  *   Item specification.
2628  * @param[in] itemFlags
2629  *   Bit-fields that holds the items detected until now.
2630  * @param[in] enPriv
2631  *   Pointer to the private data structure.
2632  * @param[out] error
2633  *   Pointer to error structure.
2634  *
2635  * @return
2636  *   0 on success, a negative errno value otherwise and rte_errno is set.
2637  */
2638
2639 int
2640 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2641                                uint64_t item_flags,
2642                                struct rte_eth_dev *dev,
2643                                struct rte_flow_error *error)
2644 {
2645         struct mlx5_priv *priv = dev->data->dev_private;
2646         const struct rte_flow_item_geneve *spec = item->spec;
2647         const struct rte_flow_item_geneve *mask = item->mask;
2648         int ret;
2649         uint16_t gbhdr;
2650         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2651                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2652         const struct rte_flow_item_geneve nic_mask = {
2653                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2654                 .vni = "\xff\xff\xff",
2655                 .protocol = RTE_BE16(UINT16_MAX),
2656         };
2657
2658         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2659                 return rte_flow_error_set(error, ENOTSUP,
2660                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2661                                           "L3 Geneve is not enabled by device"
2662                                           " parameter and/or not configured in"
2663                                           " firmware");
2664         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2665                 return rte_flow_error_set(error, ENOTSUP,
2666                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2667                                           "multiple tunnel layers not"
2668                                           " supported");
2669         /*
2670          * Verify only UDPv4 is present as defined in
2671          * https://tools.ietf.org/html/rfc7348
2672          */
2673         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2674                 return rte_flow_error_set(error, EINVAL,
2675                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2676                                           "no outer UDP layer found");
2677         if (!mask)
2678                 mask = &rte_flow_item_geneve_mask;
2679         ret = mlx5_flow_item_acceptable
2680                                   (item, (const uint8_t *)mask,
2681                                    (const uint8_t *)&nic_mask,
2682                                    sizeof(struct rte_flow_item_geneve),
2683                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2684         if (ret)
2685                 return ret;
2686         if (spec) {
2687                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2688                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2689                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2690                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2691                         return rte_flow_error_set(error, ENOTSUP,
2692                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2693                                                   item,
2694                                                   "Geneve protocol unsupported"
2695                                                   " fields are being used");
2696                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2697                         return rte_flow_error_set
2698                                         (error, ENOTSUP,
2699                                          RTE_FLOW_ERROR_TYPE_ITEM,
2700                                          item,
2701                                          "Unsupported Geneve options length");
2702         }
2703         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2704                 return rte_flow_error_set
2705                                     (error, ENOTSUP,
2706                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2707                                      "Geneve tunnel must be fully defined");
2708         return 0;
2709 }
2710
2711 /**
2712  * Validate Geneve TLV option item.
2713  *
2714  * @param[in] item
2715  *   Item specification.
2716  * @param[in] last_item
2717  *   Previous validated item in the pattern items.
2718  * @param[in] geneve_item
2719  *   Previous GENEVE item specification.
2720  * @param[in] dev
2721  *   Pointer to the rte_eth_dev structure.
2722  * @param[out] error
2723  *   Pointer to error structure.
2724  *
2725  * @return
2726  *   0 on success, a negative errno value otherwise and rte_errno is set.
2727  */
2728 int
2729 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2730                                    uint64_t last_item,
2731                                    const struct rte_flow_item *geneve_item,
2732                                    struct rte_eth_dev *dev,
2733                                    struct rte_flow_error *error)
2734 {
2735         struct mlx5_priv *priv = dev->data->dev_private;
2736         struct mlx5_dev_ctx_shared *sh = priv->sh;
2737         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2738         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2739         uint8_t data_max_supported =
2740                         hca_attr->max_geneve_tlv_option_data_len * 4;
2741         struct mlx5_dev_config *config = &priv->config;
2742         const struct rte_flow_item_geneve *geneve_spec;
2743         const struct rte_flow_item_geneve *geneve_mask;
2744         const struct rte_flow_item_geneve_opt *spec = item->spec;
2745         const struct rte_flow_item_geneve_opt *mask = item->mask;
2746         unsigned int i;
2747         unsigned int data_len;
2748         uint8_t tlv_option_len;
2749         uint16_t optlen_m, optlen_v;
2750         const struct rte_flow_item_geneve_opt full_mask = {
2751                 .option_class = RTE_BE16(0xffff),
2752                 .option_type = 0xff,
2753                 .option_len = 0x1f,
2754         };
2755
2756         if (!mask)
2757                 mask = &rte_flow_item_geneve_opt_mask;
2758         if (!spec)
2759                 return rte_flow_error_set
2760                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2761                         "Geneve TLV opt class/type/length must be specified");
2762         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2763                 return rte_flow_error_set
2764                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2765                         "Geneve TLV opt length exceeeds the limit (31)");
2766         /* Check if class type and length masks are full. */
2767         if (full_mask.option_class != mask->option_class ||
2768             full_mask.option_type != mask->option_type ||
2769             full_mask.option_len != (mask->option_len & full_mask.option_len))
2770                 return rte_flow_error_set
2771                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                         "Geneve TLV opt class/type/length masks must be full");
2773         /* Check if length is supported */
2774         if ((uint32_t)spec->option_len >
2775                         config->hca_attr.max_geneve_tlv_option_data_len)
2776                 return rte_flow_error_set
2777                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2778                         "Geneve TLV opt length not supported");
2779         if (config->hca_attr.max_geneve_tlv_options > 1)
2780                 DRV_LOG(DEBUG,
2781                         "max_geneve_tlv_options supports more than 1 option");
2782         /* Check GENEVE item preceding. */
2783         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2784                 return rte_flow_error_set
2785                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2786                         "Geneve opt item must be preceded with Geneve item");
2787         geneve_spec = geneve_item->spec;
2788         geneve_mask = geneve_item->mask ? geneve_item->mask :
2789                                           &rte_flow_item_geneve_mask;
2790         /* Check if GENEVE TLV option size doesn't exceed option length */
2791         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2792                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2793                 tlv_option_len = spec->option_len & mask->option_len;
2794                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2795                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2796                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2797                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2798                 if ((optlen_v & optlen_m) <= tlv_option_len)
2799                         return rte_flow_error_set
2800                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2801                                  "GENEVE TLV option length exceeds optlen");
2802         }
2803         /* Check if length is 0 or data is 0. */
2804         if (spec->data == NULL || spec->option_len == 0)
2805                 return rte_flow_error_set
2806                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2807                         "Geneve TLV opt with zero data/length not supported");
2808         /* Check not all data & mask are 0. */
2809         data_len = spec->option_len * 4;
2810         if (mask->data == NULL) {
2811                 for (i = 0; i < data_len; i++)
2812                         if (spec->data[i])
2813                                 break;
2814                 if (i == data_len)
2815                         return rte_flow_error_set(error, ENOTSUP,
2816                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2817                                 "Can't match on Geneve option data 0");
2818         } else {
2819                 for (i = 0; i < data_len; i++)
2820                         if (spec->data[i] & mask->data[i])
2821                                 break;
2822                 if (i == data_len)
2823                         return rte_flow_error_set(error, ENOTSUP,
2824                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2825                                 "Can't match on Geneve option data and mask 0");
2826                 /* Check data mask supported. */
2827                 for (i = data_max_supported; i < data_len ; i++)
2828                         if (mask->data[i])
2829                                 return rte_flow_error_set(error, ENOTSUP,
2830                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2831                                         "Data mask is of unsupported size");
2832         }
2833         /* Check GENEVE option is supported in NIC. */
2834         if (!config->hca_attr.geneve_tlv_opt)
2835                 return rte_flow_error_set
2836                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2837                         "Geneve TLV opt not supported");
2838         /* Check if we already have geneve option with different type/class. */
2839         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2840         geneve_opt_resource = sh->geneve_tlv_option_resource;
2841         if (geneve_opt_resource != NULL)
2842                 if (geneve_opt_resource->option_class != spec->option_class ||
2843                     geneve_opt_resource->option_type != spec->option_type ||
2844                     geneve_opt_resource->length != spec->option_len) {
2845                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2846                         return rte_flow_error_set(error, ENOTSUP,
2847                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2848                                 "Only one Geneve TLV option supported");
2849                 }
2850         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2851         return 0;
2852 }
2853
2854 /**
2855  * Validate MPLS item.
2856  *
2857  * @param[in] dev
2858  *   Pointer to the rte_eth_dev structure.
2859  * @param[in] item
2860  *   Item specification.
2861  * @param[in] item_flags
2862  *   Bit-fields that holds the items detected until now.
2863  * @param[in] prev_layer
2864  *   The protocol layer indicated in previous item.
2865  * @param[out] error
2866  *   Pointer to error structure.
2867  *
2868  * @return
2869  *   0 on success, a negative errno value otherwise and rte_errno is set.
2870  */
2871 int
2872 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2873                              const struct rte_flow_item *item __rte_unused,
2874                              uint64_t item_flags __rte_unused,
2875                              uint64_t prev_layer __rte_unused,
2876                              struct rte_flow_error *error)
2877 {
2878 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2879         const struct rte_flow_item_mpls *mask = item->mask;
2880         struct mlx5_priv *priv = dev->data->dev_private;
2881         int ret;
2882
2883         if (!priv->config.mpls_en)
2884                 return rte_flow_error_set(error, ENOTSUP,
2885                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2886                                           "MPLS not supported or"
2887                                           " disabled in firmware"
2888                                           " configuration.");
2889         /* MPLS over IP, UDP, GRE is allowed */
2890         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2891                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2892                             MLX5_FLOW_LAYER_GRE |
2893                             MLX5_FLOW_LAYER_GRE_KEY)))
2894                 return rte_flow_error_set(error, EINVAL,
2895                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2896                                           "protocol filtering not compatible"
2897                                           " with MPLS layer");
2898         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2899         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2900             !(item_flags & MLX5_FLOW_LAYER_GRE))
2901                 return rte_flow_error_set(error, ENOTSUP,
2902                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2903                                           "multiple tunnel layers not"
2904                                           " supported");
2905         if (!mask)
2906                 mask = &rte_flow_item_mpls_mask;
2907         ret = mlx5_flow_item_acceptable
2908                 (item, (const uint8_t *)mask,
2909                  (const uint8_t *)&rte_flow_item_mpls_mask,
2910                  sizeof(struct rte_flow_item_mpls),
2911                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2912         if (ret < 0)
2913                 return ret;
2914         return 0;
2915 #else
2916         return rte_flow_error_set(error, ENOTSUP,
2917                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2918                                   "MPLS is not supported by Verbs, please"
2919                                   " update.");
2920 #endif
2921 }
2922
2923 /**
2924  * Validate NVGRE item.
2925  *
2926  * @param[in] item
2927  *   Item specification.
2928  * @param[in] item_flags
2929  *   Bit flags to mark detected items.
2930  * @param[in] target_protocol
2931  *   The next protocol in the previous item.
2932  * @param[out] error
2933  *   Pointer to error structure.
2934  *
2935  * @return
2936  *   0 on success, a negative errno value otherwise and rte_errno is set.
2937  */
2938 int
2939 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2940                               uint64_t item_flags,
2941                               uint8_t target_protocol,
2942                               struct rte_flow_error *error)
2943 {
2944         const struct rte_flow_item_nvgre *mask = item->mask;
2945         int ret;
2946
2947         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2948                 return rte_flow_error_set(error, EINVAL,
2949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2950                                           "protocol filtering not compatible"
2951                                           " with this GRE layer");
2952         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2953                 return rte_flow_error_set(error, ENOTSUP,
2954                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2955                                           "multiple tunnel layers not"
2956                                           " supported");
2957         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2958                 return rte_flow_error_set(error, ENOTSUP,
2959                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2960                                           "L3 Layer is missing");
2961         if (!mask)
2962                 mask = &rte_flow_item_nvgre_mask;
2963         ret = mlx5_flow_item_acceptable
2964                 (item, (const uint8_t *)mask,
2965                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2966                  sizeof(struct rte_flow_item_nvgre),
2967                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2968         if (ret < 0)
2969                 return ret;
2970         return 0;
2971 }
2972
2973 /**
2974  * Validate eCPRI item.
2975  *
2976  * @param[in] item
2977  *   Item specification.
2978  * @param[in] item_flags
2979  *   Bit-fields that holds the items detected until now.
2980  * @param[in] last_item
2981  *   Previous validated item in the pattern items.
2982  * @param[in] ether_type
2983  *   Type in the ethernet layer header (including dot1q).
2984  * @param[in] acc_mask
2985  *   Acceptable mask, if NULL default internal default mask
2986  *   will be used to check whether item fields are supported.
2987  * @param[out] error
2988  *   Pointer to error structure.
2989  *
2990  * @return
2991  *   0 on success, a negative errno value otherwise and rte_errno is set.
2992  */
2993 int
2994 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2995                               uint64_t item_flags,
2996                               uint64_t last_item,
2997                               uint16_t ether_type,
2998                               const struct rte_flow_item_ecpri *acc_mask,
2999                               struct rte_flow_error *error)
3000 {
3001         const struct rte_flow_item_ecpri *mask = item->mask;
3002         const struct rte_flow_item_ecpri nic_mask = {
3003                 .hdr = {
3004                         .common = {
3005                                 .u32 =
3006                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3007                                         .type = 0xFF,
3008                                         }).u32),
3009                         },
3010                         .dummy[0] = 0xFFFFFFFF,
3011                 },
3012         };
3013         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3014                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3015         struct rte_flow_item_ecpri mask_lo;
3016
3017         if (!(last_item & outer_l2_vlan) &&
3018             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3019                 return rte_flow_error_set(error, EINVAL,
3020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3021                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3022         if ((last_item & outer_l2_vlan) && ether_type &&
3023             ether_type != RTE_ETHER_TYPE_ECPRI)
3024                 return rte_flow_error_set(error, EINVAL,
3025                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3026                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3027         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3028                 return rte_flow_error_set(error, EINVAL,
3029                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3030                                           "eCPRI with tunnel is not supported right now");
3031         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3032                 return rte_flow_error_set(error, ENOTSUP,
3033                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3034                                           "multiple L3 layers not supported");
3035         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3036                 return rte_flow_error_set(error, EINVAL,
3037                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3038                                           "eCPRI cannot coexist with a TCP layer");
3039         /* In specification, eCPRI could be over UDP layer. */
3040         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3041                 return rte_flow_error_set(error, EINVAL,
3042                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3043                                           "eCPRI over UDP layer is not yet supported right now");
3044         /* Mask for type field in common header could be zero. */
3045         if (!mask)
3046                 mask = &rte_flow_item_ecpri_mask;
3047         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3048         /* Input mask is in big-endian format. */
3049         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3050                 return rte_flow_error_set(error, EINVAL,
3051                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3052                                           "partial mask is not supported for protocol");
3053         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3054                 return rte_flow_error_set(error, EINVAL,
3055                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3056                                           "message header mask must be after a type mask");
3057         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3058                                          acc_mask ? (const uint8_t *)acc_mask
3059                                                   : (const uint8_t *)&nic_mask,
3060                                          sizeof(struct rte_flow_item_ecpri),
3061                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3062 }
3063
3064 /**
3065  * Release resource related QUEUE/RSS action split.
3066  *
3067  * @param dev
3068  *   Pointer to Ethernet device.
3069  * @param flow
3070  *   Flow to release id's from.
3071  */
3072 static void
3073 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
3074                              struct rte_flow *flow)
3075 {
3076         struct mlx5_priv *priv = dev->data->dev_private;
3077         uint32_t handle_idx;
3078         struct mlx5_flow_handle *dev_handle;
3079
3080         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
3081                        handle_idx, dev_handle, next)
3082                 if (dev_handle->split_flow_id &&
3083                     !dev_handle->is_meter_flow_id)
3084                         mlx5_ipool_free(priv->sh->ipool
3085                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
3086                                         dev_handle->split_flow_id);
3087 }
3088
3089 static int
3090 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3091                    const struct rte_flow_attr *attr __rte_unused,
3092                    const struct rte_flow_item items[] __rte_unused,
3093                    const struct rte_flow_action actions[] __rte_unused,
3094                    bool external __rte_unused,
3095                    int hairpin __rte_unused,
3096                    struct rte_flow_error *error)
3097 {
3098         return rte_flow_error_set(error, ENOTSUP,
3099                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3100 }
3101
3102 static struct mlx5_flow *
3103 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3104                   const struct rte_flow_attr *attr __rte_unused,
3105                   const struct rte_flow_item items[] __rte_unused,
3106                   const struct rte_flow_action actions[] __rte_unused,
3107                   struct rte_flow_error *error)
3108 {
3109         rte_flow_error_set(error, ENOTSUP,
3110                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3111         return NULL;
3112 }
3113
3114 static int
3115 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3116                     struct mlx5_flow *dev_flow __rte_unused,
3117                     const struct rte_flow_attr *attr __rte_unused,
3118                     const struct rte_flow_item items[] __rte_unused,
3119                     const struct rte_flow_action actions[] __rte_unused,
3120                     struct rte_flow_error *error)
3121 {
3122         return rte_flow_error_set(error, ENOTSUP,
3123                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3124 }
3125
3126 static int
3127 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3128                 struct rte_flow *flow __rte_unused,
3129                 struct rte_flow_error *error)
3130 {
3131         return rte_flow_error_set(error, ENOTSUP,
3132                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3133 }
3134
3135 static void
3136 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3137                  struct rte_flow *flow __rte_unused)
3138 {
3139 }
3140
3141 static void
3142 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3143                   struct rte_flow *flow __rte_unused)
3144 {
3145 }
3146
3147 static int
3148 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3149                 struct rte_flow *flow __rte_unused,
3150                 const struct rte_flow_action *actions __rte_unused,
3151                 void *data __rte_unused,
3152                 struct rte_flow_error *error)
3153 {
3154         return rte_flow_error_set(error, ENOTSUP,
3155                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3156 }
3157
3158 static int
3159 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3160                       uint32_t domains __rte_unused,
3161                       uint32_t flags __rte_unused)
3162 {
3163         return 0;
3164 }
3165
3166 /* Void driver to protect from null pointer reference. */
3167 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3168         .validate = flow_null_validate,
3169         .prepare = flow_null_prepare,
3170         .translate = flow_null_translate,
3171         .apply = flow_null_apply,
3172         .remove = flow_null_remove,
3173         .destroy = flow_null_destroy,
3174         .query = flow_null_query,
3175         .sync_domain = flow_null_sync_domain,
3176 };
3177
3178 /**
3179  * Select flow driver type according to flow attributes and device
3180  * configuration.
3181  *
3182  * @param[in] dev
3183  *   Pointer to the dev structure.
3184  * @param[in] attr
3185  *   Pointer to the flow attributes.
3186  *
3187  * @return
3188  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3189  */
3190 static enum mlx5_flow_drv_type
3191 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3192 {
3193         struct mlx5_priv *priv = dev->data->dev_private;
3194         /* The OS can determine first a specific flow type (DV, VERBS) */
3195         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3196
3197         if (type != MLX5_FLOW_TYPE_MAX)
3198                 return type;
3199         /* If no OS specific type - continue with DV/VERBS selection */
3200         if (attr->transfer && priv->config.dv_esw_en)
3201                 type = MLX5_FLOW_TYPE_DV;
3202         if (!attr->transfer)
3203                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3204                                                  MLX5_FLOW_TYPE_VERBS;
3205         return type;
3206 }
3207
3208 #define flow_get_drv_ops(type) flow_drv_ops[type]
3209
3210 /**
3211  * Flow driver validation API. This abstracts calling driver specific functions.
3212  * The type of flow driver is determined according to flow attributes.
3213  *
3214  * @param[in] dev
3215  *   Pointer to the dev structure.
3216  * @param[in] attr
3217  *   Pointer to the flow attributes.
3218  * @param[in] items
3219  *   Pointer to the list of items.
3220  * @param[in] actions
3221  *   Pointer to the list of actions.
3222  * @param[in] external
3223  *   This flow rule is created by request external to PMD.
3224  * @param[in] hairpin
3225  *   Number of hairpin TX actions, 0 means classic flow.
3226  * @param[out] error
3227  *   Pointer to the error structure.
3228  *
3229  * @return
3230  *   0 on success, a negative errno value otherwise and rte_errno is set.
3231  */
3232 static inline int
3233 flow_drv_validate(struct rte_eth_dev *dev,
3234                   const struct rte_flow_attr *attr,
3235                   const struct rte_flow_item items[],
3236                   const struct rte_flow_action actions[],
3237                   bool external, int hairpin, struct rte_flow_error *error)
3238 {
3239         const struct mlx5_flow_driver_ops *fops;
3240         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3241
3242         fops = flow_get_drv_ops(type);
3243         return fops->validate(dev, attr, items, actions, external,
3244                               hairpin, error);
3245 }
3246
3247 /**
3248  * Flow driver preparation API. This abstracts calling driver specific
3249  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3250  * calculates the size of memory required for device flow, allocates the memory,
3251  * initializes the device flow and returns the pointer.
3252  *
3253  * @note
3254  *   This function initializes device flow structure such as dv or verbs in
3255  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3256  *   rest. For example, adding returning device flow to flow->dev_flow list and
3257  *   setting backward reference to the flow should be done out of this function.
3258  *   layers field is not filled either.
3259  *
3260  * @param[in] dev
3261  *   Pointer to the dev structure.
3262  * @param[in] attr
3263  *   Pointer to the flow attributes.
3264  * @param[in] items
3265  *   Pointer to the list of items.
3266  * @param[in] actions
3267  *   Pointer to the list of actions.
3268  * @param[in] flow_idx
3269  *   This memory pool index to the flow.
3270  * @param[out] error
3271  *   Pointer to the error structure.
3272  *
3273  * @return
3274  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3275  */
3276 static inline struct mlx5_flow *
3277 flow_drv_prepare(struct rte_eth_dev *dev,
3278                  const struct rte_flow *flow,
3279                  const struct rte_flow_attr *attr,
3280                  const struct rte_flow_item items[],
3281                  const struct rte_flow_action actions[],
3282                  uint32_t flow_idx,
3283                  struct rte_flow_error *error)
3284 {
3285         const struct mlx5_flow_driver_ops *fops;
3286         enum mlx5_flow_drv_type type = flow->drv_type;
3287         struct mlx5_flow *mlx5_flow = NULL;
3288
3289         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3290         fops = flow_get_drv_ops(type);
3291         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3292         if (mlx5_flow)
3293                 mlx5_flow->flow_idx = flow_idx;
3294         return mlx5_flow;
3295 }
3296
3297 /**
3298  * Flow driver translation API. This abstracts calling driver specific
3299  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3300  * translates a generic flow into a driver flow. flow_drv_prepare() must
3301  * precede.
3302  *
3303  * @note
3304  *   dev_flow->layers could be filled as a result of parsing during translation
3305  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3306  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3307  *   flow->actions could be overwritten even though all the expanded dev_flows
3308  *   have the same actions.
3309  *
3310  * @param[in] dev
3311  *   Pointer to the rte dev structure.
3312  * @param[in, out] dev_flow
3313  *   Pointer to the mlx5 flow.
3314  * @param[in] attr
3315  *   Pointer to the flow attributes.
3316  * @param[in] items
3317  *   Pointer to the list of items.
3318  * @param[in] actions
3319  *   Pointer to the list of actions.
3320  * @param[out] error
3321  *   Pointer to the error structure.
3322  *
3323  * @return
3324  *   0 on success, a negative errno value otherwise and rte_errno is set.
3325  */
3326 static inline int
3327 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3328                    const struct rte_flow_attr *attr,
3329                    const struct rte_flow_item items[],
3330                    const struct rte_flow_action actions[],
3331                    struct rte_flow_error *error)
3332 {
3333         const struct mlx5_flow_driver_ops *fops;
3334         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3335
3336         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3337         fops = flow_get_drv_ops(type);
3338         return fops->translate(dev, dev_flow, attr, items, actions, error);
3339 }
3340
3341 /**
3342  * Flow driver apply API. This abstracts calling driver specific functions.
3343  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3344  * translated driver flows on to device. flow_drv_translate() must precede.
3345  *
3346  * @param[in] dev
3347  *   Pointer to Ethernet device structure.
3348  * @param[in, out] flow
3349  *   Pointer to flow structure.
3350  * @param[out] error
3351  *   Pointer to error structure.
3352  *
3353  * @return
3354  *   0 on success, a negative errno value otherwise and rte_errno is set.
3355  */
3356 static inline int
3357 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3358                struct rte_flow_error *error)
3359 {
3360         const struct mlx5_flow_driver_ops *fops;
3361         enum mlx5_flow_drv_type type = flow->drv_type;
3362
3363         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3364         fops = flow_get_drv_ops(type);
3365         return fops->apply(dev, flow, error);
3366 }
3367
3368 /**
3369  * Flow driver destroy API. This abstracts calling driver specific functions.
3370  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3371  * on device and releases resources of the flow.
3372  *
3373  * @param[in] dev
3374  *   Pointer to Ethernet device.
3375  * @param[in, out] flow
3376  *   Pointer to flow structure.
3377  */
3378 static inline void
3379 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3380 {
3381         const struct mlx5_flow_driver_ops *fops;
3382         enum mlx5_flow_drv_type type = flow->drv_type;
3383
3384         flow_mreg_split_qrss_release(dev, flow);
3385         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3386         fops = flow_get_drv_ops(type);
3387         fops->destroy(dev, flow);
3388 }
3389
3390 /**
3391  * Flow driver find RSS policy tbl API. This abstracts calling driver
3392  * specific functions. Parent flow (rte_flow) should have driver
3393  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3394  *
3395  * @param[in] dev
3396  *   Pointer to Ethernet device.
3397  * @param[in, out] flow
3398  *   Pointer to flow structure.
3399  * @param[in] policy
3400  *   Pointer to meter policy table.
3401  * @param[in] rss_desc
3402  *   Pointer to rss_desc
3403  */
3404 static struct mlx5_flow_meter_sub_policy *
3405 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3406                 struct rte_flow *flow,
3407                 struct mlx5_flow_meter_policy *policy,
3408                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3409 {
3410         const struct mlx5_flow_driver_ops *fops;
3411         enum mlx5_flow_drv_type type = flow->drv_type;
3412
3413         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3414         fops = flow_get_drv_ops(type);
3415         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3416 }
3417
3418 /**
3419  * Get RSS action from the action list.
3420  *
3421  * @param[in] dev
3422  *   Pointer to Ethernet device.
3423  * @param[in] actions
3424  *   Pointer to the list of actions.
3425  * @param[in] flow
3426  *   Parent flow structure pointer.
3427  *
3428  * @return
3429  *   Pointer to the RSS action if exist, else return NULL.
3430  */
3431 static const struct rte_flow_action_rss*
3432 flow_get_rss_action(struct rte_eth_dev *dev,
3433                     const struct rte_flow_action actions[])
3434 {
3435         struct mlx5_priv *priv = dev->data->dev_private;
3436         const struct rte_flow_action_rss *rss = NULL;
3437
3438         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3439                 switch (actions->type) {
3440                 case RTE_FLOW_ACTION_TYPE_RSS:
3441                         rss = actions->conf;
3442                         break;
3443                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3444                 {
3445                         const struct rte_flow_action_sample *sample =
3446                                                                 actions->conf;
3447                         const struct rte_flow_action *act = sample->actions;
3448                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3449                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3450                                         rss = act->conf;
3451                         break;
3452                 }
3453                 case RTE_FLOW_ACTION_TYPE_METER:
3454                 {
3455                         uint32_t mtr_idx;
3456                         struct mlx5_flow_meter_info *fm;
3457                         struct mlx5_flow_meter_policy *policy;
3458                         const struct rte_flow_action_meter *mtr = actions->conf;
3459
3460                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3461                         if (fm) {
3462                                 policy = mlx5_flow_meter_policy_find(dev,
3463                                                 fm->policy_id, NULL);
3464                                 if (policy && policy->is_rss)
3465                                         rss =
3466                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3467                         }
3468                         break;
3469                 }
3470                 default:
3471                         break;
3472                 }
3473         }
3474         return rss;
3475 }
3476
3477 /**
3478  * Get ASO age action by index.
3479  *
3480  * @param[in] dev
3481  *   Pointer to the Ethernet device structure.
3482  * @param[in] age_idx
3483  *   Index to the ASO age action.
3484  *
3485  * @return
3486  *   The specified ASO age action.
3487  */
3488 struct mlx5_aso_age_action*
3489 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3490 {
3491         uint16_t pool_idx = age_idx & UINT16_MAX;
3492         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3493         struct mlx5_priv *priv = dev->data->dev_private;
3494         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3495         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3496
3497         return &pool->actions[offset - 1];
3498 }
3499
3500 /* maps indirect action to translated direct in some actions array */
3501 struct mlx5_translated_action_handle {
3502         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3503         int index; /**< Index in related array of rte_flow_action. */
3504 };
3505
3506 /**
3507  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3508  * direct action if translation possible.
3509  * This functionality used to run same execution path for both direct and
3510  * indirect actions on flow create. All necessary preparations for indirect
3511  * action handling should be performed on *handle* actions list returned
3512  * from this call.
3513  *
3514  * @param[in] dev
3515  *   Pointer to Ethernet device.
3516  * @param[in] actions
3517  *   List of actions to translate.
3518  * @param[out] handle
3519  *   List to store translated indirect action object handles.
3520  * @param[in, out] indir_n
3521  *   Size of *handle* array. On return should be updated with number of
3522  *   indirect actions retrieved from the *actions* list.
3523  * @param[out] translated_actions
3524  *   List of actions where all indirect actions were translated to direct
3525  *   if possible. NULL if no translation took place.
3526  * @param[out] error
3527  *   Pointer to the error structure.
3528  *
3529  * @return
3530  *   0 on success, a negative errno value otherwise and rte_errno is set.
3531  */
3532 static int
3533 flow_action_handles_translate(struct rte_eth_dev *dev,
3534                               const struct rte_flow_action actions[],
3535                               struct mlx5_translated_action_handle *handle,
3536                               int *indir_n,
3537                               struct rte_flow_action **translated_actions,
3538                               struct rte_flow_error *error)
3539 {
3540         struct mlx5_priv *priv = dev->data->dev_private;
3541         struct rte_flow_action *translated = NULL;
3542         size_t actions_size;
3543         int n;
3544         int copied_n = 0;
3545         struct mlx5_translated_action_handle *handle_end = NULL;
3546
3547         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3548                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3549                         continue;
3550                 if (copied_n == *indir_n) {
3551                         return rte_flow_error_set
3552                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3553                                  NULL, "too many shared actions");
3554                 }
3555                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3556                            sizeof(actions[n].conf));
3557                 handle[copied_n].index = n;
3558                 copied_n++;
3559         }
3560         n++;
3561         *indir_n = copied_n;
3562         if (!copied_n)
3563                 return 0;
3564         actions_size = sizeof(struct rte_flow_action) * n;
3565         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3566         if (!translated) {
3567                 rte_errno = ENOMEM;
3568                 return -ENOMEM;
3569         }
3570         memcpy(translated, actions, actions_size);
3571         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3572                 struct mlx5_shared_action_rss *shared_rss;
3573                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3574                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3575                 uint32_t idx = act_idx &
3576                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3577
3578                 switch (type) {
3579                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3580                         shared_rss = mlx5_ipool_get
3581                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3582                         translated[handle->index].type =
3583                                 RTE_FLOW_ACTION_TYPE_RSS;
3584                         translated[handle->index].conf =
3585                                 &shared_rss->origin;
3586                         break;
3587                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3588                         translated[handle->index].type =
3589                                                 (enum rte_flow_action_type)
3590                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3591                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3592                         break;
3593                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3594                         if (priv->sh->flow_hit_aso_en) {
3595                                 translated[handle->index].type =
3596                                         (enum rte_flow_action_type)
3597                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3598                                 translated[handle->index].conf =
3599                                                          (void *)(uintptr_t)idx;
3600                                 break;
3601                         }
3602                         /* Fall-through */
3603                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3604                         if (priv->sh->ct_aso_en) {
3605                                 translated[handle->index].type =
3606                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3607                                 translated[handle->index].conf =
3608                                                          (void *)(uintptr_t)idx;
3609                                 break;
3610                         }
3611                         /* Fall-through */
3612                 default:
3613                         mlx5_free(translated);
3614                         return rte_flow_error_set
3615                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3616                                  NULL, "invalid indirect action type");
3617                 }
3618         }
3619         *translated_actions = translated;
3620         return 0;
3621 }
3622
3623 /**
3624  * Get Shared RSS action from the action list.
3625  *
3626  * @param[in] dev
3627  *   Pointer to Ethernet device.
3628  * @param[in] shared
3629  *   Pointer to the list of actions.
3630  * @param[in] shared_n
3631  *   Actions list length.
3632  *
3633  * @return
3634  *   The MLX5 RSS action ID if exists, otherwise return 0.
3635  */
3636 static uint32_t
3637 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3638                            struct mlx5_translated_action_handle *handle,
3639                            int shared_n)
3640 {
3641         struct mlx5_translated_action_handle *handle_end;
3642         struct mlx5_priv *priv = dev->data->dev_private;
3643         struct mlx5_shared_action_rss *shared_rss;
3644
3645
3646         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3647                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3648                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3649                 uint32_t idx = act_idx &
3650                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3651                 switch (type) {
3652                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3653                         shared_rss = mlx5_ipool_get
3654                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3655                                                                            idx);
3656                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3657                                            __ATOMIC_RELAXED);
3658                         return idx;
3659                 default:
3660                         break;
3661                 }
3662         }
3663         return 0;
3664 }
3665
3666 static unsigned int
3667 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3668 {
3669         const struct rte_flow_item *item;
3670         unsigned int has_vlan = 0;
3671
3672         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3673                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3674                         has_vlan = 1;
3675                         break;
3676                 }
3677         }
3678         if (has_vlan)
3679                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3680                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3681         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3682                                MLX5_EXPANSION_ROOT_OUTER;
3683 }
3684
3685 /**
3686  *  Get layer flags from the prefix flow.
3687  *
3688  *  Some flows may be split to several subflows, the prefix subflow gets the
3689  *  match items and the suffix sub flow gets the actions.
3690  *  Some actions need the user defined match item flags to get the detail for
3691  *  the action.
3692  *  This function helps the suffix flow to get the item layer flags from prefix
3693  *  subflow.
3694  *
3695  * @param[in] dev_flow
3696  *   Pointer the created preifx subflow.
3697  *
3698  * @return
3699  *   The layers get from prefix subflow.
3700  */
3701 static inline uint64_t
3702 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3703 {
3704         uint64_t layers = 0;
3705
3706         /*
3707          * Layers bits could be localization, but usually the compiler will
3708          * help to do the optimization work for source code.
3709          * If no decap actions, use the layers directly.
3710          */
3711         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3712                 return dev_flow->handle->layers;
3713         /* Convert L3 layers with decap action. */
3714         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3715                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3716         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3717                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3718         /* Convert L4 layers with decap action.  */
3719         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3720                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3721         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3722                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3723         return layers;
3724 }
3725
3726 /**
3727  * Get metadata split action information.
3728  *
3729  * @param[in] actions
3730  *   Pointer to the list of actions.
3731  * @param[out] qrss
3732  *   Pointer to the return pointer.
3733  * @param[out] qrss_type
3734  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3735  *   if no QUEUE/RSS is found.
3736  * @param[out] encap_idx
3737  *   Pointer to the index of the encap action if exists, otherwise the last
3738  *   action index.
3739  *
3740  * @return
3741  *   Total number of actions.
3742  */
3743 static int
3744 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3745                                        const struct rte_flow_action **qrss,
3746                                        int *encap_idx)
3747 {
3748         const struct rte_flow_action_raw_encap *raw_encap;
3749         int actions_n = 0;
3750         int raw_decap_idx = -1;
3751
3752         *encap_idx = -1;
3753         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3754                 switch (actions->type) {
3755                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3756                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3757                         *encap_idx = actions_n;
3758                         break;
3759                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3760                         raw_decap_idx = actions_n;
3761                         break;
3762                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3763                         raw_encap = actions->conf;
3764                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3765                                 *encap_idx = raw_decap_idx != -1 ?
3766                                                       raw_decap_idx : actions_n;
3767                         break;
3768                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3769                 case RTE_FLOW_ACTION_TYPE_RSS:
3770                         *qrss = actions;
3771                         break;
3772                 default:
3773                         break;
3774                 }
3775                 actions_n++;
3776         }
3777         if (*encap_idx == -1)
3778                 *encap_idx = actions_n;
3779         /* Count RTE_FLOW_ACTION_TYPE_END. */
3780         return actions_n + 1;
3781 }
3782
3783 /**
3784  * Check if the action will change packet.
3785  *
3786  * @param dev
3787  *   Pointer to Ethernet device.
3788  * @param[in] type
3789  *   action type.
3790  *
3791  * @return
3792  *   true if action will change packet, false otherwise.
3793  */
3794 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3795                                           enum rte_flow_action_type type)
3796 {
3797         struct mlx5_priv *priv = dev->data->dev_private;
3798
3799         switch (type) {
3800         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3801         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3802         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3803         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3804         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3805         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3806         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3807         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3808         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3809         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3810         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3811         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3812         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3813         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3814         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3815         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3816         case RTE_FLOW_ACTION_TYPE_SET_META:
3817         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3818         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3819         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3820         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3821         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3822         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3823         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3824         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3825         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3826         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3827         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3828         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3829                 return true;
3830         case RTE_FLOW_ACTION_TYPE_FLAG:
3831         case RTE_FLOW_ACTION_TYPE_MARK:
3832                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3833                         return true;
3834                 else
3835                         return false;
3836         default:
3837                 return false;
3838         }
3839 }
3840
3841 /**
3842  * Check meter action from the action list.
3843  *
3844  * @param dev
3845  *   Pointer to Ethernet device.
3846  * @param[in] actions
3847  *   Pointer to the list of actions.
3848  * @param[out] has_mtr
3849  *   Pointer to the meter exist flag.
3850  * @param[out] has_modify
3851  *   Pointer to the flag showing there's packet change action.
3852  * @param[out] meter_id
3853  *   Pointer to the meter id.
3854  *
3855  * @return
3856  *   Total number of actions.
3857  */
3858 static int
3859 flow_check_meter_action(struct rte_eth_dev *dev,
3860                         const struct rte_flow_action actions[],
3861                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3862 {
3863         const struct rte_flow_action_meter *mtr = NULL;
3864         int actions_n = 0;
3865
3866         MLX5_ASSERT(has_mtr);
3867         *has_mtr = false;
3868         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3869                 switch (actions->type) {
3870                 case RTE_FLOW_ACTION_TYPE_METER:
3871                         mtr = actions->conf;
3872                         *meter_id = mtr->mtr_id;
3873                         *has_mtr = true;
3874                         break;
3875                 default:
3876                         break;
3877                 }
3878                 if (!*has_mtr)
3879                         *has_modify |= flow_check_modify_action_type(dev,
3880                                                                 actions->type);
3881                 actions_n++;
3882         }
3883         /* Count RTE_FLOW_ACTION_TYPE_END. */
3884         return actions_n + 1;
3885 }
3886
3887 /**
3888  * Check if the flow should be split due to hairpin.
3889  * The reason for the split is that in current HW we can't
3890  * support encap and push-vlan on Rx, so if a flow contains
3891  * these actions we move it to Tx.
3892  *
3893  * @param dev
3894  *   Pointer to Ethernet device.
3895  * @param[in] attr
3896  *   Flow rule attributes.
3897  * @param[in] actions
3898  *   Associated actions (list terminated by the END action).
3899  *
3900  * @return
3901  *   > 0 the number of actions and the flow should be split,
3902  *   0 when no split required.
3903  */
3904 static int
3905 flow_check_hairpin_split(struct rte_eth_dev *dev,
3906                          const struct rte_flow_attr *attr,
3907                          const struct rte_flow_action actions[])
3908 {
3909         int queue_action = 0;
3910         int action_n = 0;
3911         int split = 0;
3912         const struct rte_flow_action_queue *queue;
3913         const struct rte_flow_action_rss *rss;
3914         const struct rte_flow_action_raw_encap *raw_encap;
3915         const struct rte_eth_hairpin_conf *conf;
3916
3917         if (!attr->ingress)
3918                 return 0;
3919         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3920                 switch (actions->type) {
3921                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3922                         queue = actions->conf;
3923                         if (queue == NULL)
3924                                 return 0;
3925                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3926                         if (conf == NULL || conf->tx_explicit != 0)
3927                                 return 0;
3928                         queue_action = 1;
3929                         action_n++;
3930                         break;
3931                 case RTE_FLOW_ACTION_TYPE_RSS:
3932                         rss = actions->conf;
3933                         if (rss == NULL || rss->queue_num == 0)
3934                                 return 0;
3935                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3936                         if (conf == NULL || conf->tx_explicit != 0)
3937                                 return 0;
3938                         queue_action = 1;
3939                         action_n++;
3940                         break;
3941                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3942                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3943                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3944                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3945                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3946                         split++;
3947                         action_n++;
3948                         break;
3949                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3950                         raw_encap = actions->conf;
3951                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3952                                 split++;
3953                         action_n++;
3954                         break;
3955                 default:
3956                         action_n++;
3957                         break;
3958                 }
3959         }
3960         if (split && queue_action)
3961                 return action_n;
3962         return 0;
3963 }
3964
3965 /* Declare flow create/destroy prototype in advance. */
3966 static uint32_t
3967 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3968                  const struct rte_flow_attr *attr,
3969                  const struct rte_flow_item items[],
3970                  const struct rte_flow_action actions[],
3971                  bool external, struct rte_flow_error *error);
3972
3973 static void
3974 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3975                   uint32_t flow_idx);
3976
3977 int
3978 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
3979                       struct mlx5_hlist_entry *entry,
3980                       uint64_t key, void *cb_ctx __rte_unused)
3981 {
3982         struct mlx5_flow_mreg_copy_resource *mcp_res =
3983                 container_of(entry, typeof(*mcp_res), hlist_ent);
3984
3985         return mcp_res->mark_id != key;
3986 }
3987
3988 struct mlx5_hlist_entry *
3989 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3990                        void *cb_ctx)
3991 {
3992         struct rte_eth_dev *dev = list->ctx;
3993         struct mlx5_priv *priv = dev->data->dev_private;
3994         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3995         struct mlx5_flow_mreg_copy_resource *mcp_res;
3996         struct rte_flow_error *error = ctx->error;
3997         uint32_t idx = 0;
3998         int ret;
3999         uint32_t mark_id = key;
4000         struct rte_flow_attr attr = {
4001                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4002                 .ingress = 1,
4003         };
4004         struct mlx5_rte_flow_item_tag tag_spec = {
4005                 .data = mark_id,
4006         };
4007         struct rte_flow_item items[] = {
4008                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4009         };
4010         struct rte_flow_action_mark ftag = {
4011                 .id = mark_id,
4012         };
4013         struct mlx5_flow_action_copy_mreg cp_mreg = {
4014                 .dst = REG_B,
4015                 .src = REG_NON,
4016         };
4017         struct rte_flow_action_jump jump = {
4018                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4019         };
4020         struct rte_flow_action actions[] = {
4021                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4022         };
4023
4024         /* Fill the register fileds in the flow. */
4025         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4026         if (ret < 0)
4027                 return NULL;
4028         tag_spec.id = ret;
4029         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4030         if (ret < 0)
4031                 return NULL;
4032         cp_mreg.src = ret;
4033         /* Provide the full width of FLAG specific value. */
4034         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4035                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4036         /* Build a new flow. */
4037         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4038                 items[0] = (struct rte_flow_item){
4039                         .type = (enum rte_flow_item_type)
4040                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4041                         .spec = &tag_spec,
4042                 };
4043                 items[1] = (struct rte_flow_item){
4044                         .type = RTE_FLOW_ITEM_TYPE_END,
4045                 };
4046                 actions[0] = (struct rte_flow_action){
4047                         .type = (enum rte_flow_action_type)
4048                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4049                         .conf = &ftag,
4050                 };
4051                 actions[1] = (struct rte_flow_action){
4052                         .type = (enum rte_flow_action_type)
4053                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4054                         .conf = &cp_mreg,
4055                 };
4056                 actions[2] = (struct rte_flow_action){
4057                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4058                         .conf = &jump,
4059                 };
4060                 actions[3] = (struct rte_flow_action){
4061                         .type = RTE_FLOW_ACTION_TYPE_END,
4062                 };
4063         } else {
4064                 /* Default rule, wildcard match. */
4065                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4066                 items[0] = (struct rte_flow_item){
4067                         .type = RTE_FLOW_ITEM_TYPE_END,
4068                 };
4069                 actions[0] = (struct rte_flow_action){
4070                         .type = (enum rte_flow_action_type)
4071                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4072                         .conf = &cp_mreg,
4073                 };
4074                 actions[1] = (struct rte_flow_action){
4075                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4076                         .conf = &jump,
4077                 };
4078                 actions[2] = (struct rte_flow_action){
4079                         .type = RTE_FLOW_ACTION_TYPE_END,
4080                 };
4081         }
4082         /* Build a new entry. */
4083         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4084         if (!mcp_res) {
4085                 rte_errno = ENOMEM;
4086                 return NULL;
4087         }
4088         mcp_res->idx = idx;
4089         mcp_res->mark_id = mark_id;
4090         /*
4091          * The copy Flows are not included in any list. There
4092          * ones are referenced from other Flows and can not
4093          * be applied, removed, deleted in ardbitrary order
4094          * by list traversing.
4095          */
4096         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
4097                                          actions, false, error);
4098         if (!mcp_res->rix_flow) {
4099                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4100                 return NULL;
4101         }
4102         return &mcp_res->hlist_ent;
4103 }
4104
4105 /**
4106  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4107  *
4108  * As mark_id is unique, if there's already a registered flow for the mark_id,
4109  * return by increasing the reference counter of the resource. Otherwise, create
4110  * the resource (mcp_res) and flow.
4111  *
4112  * Flow looks like,
4113  *   - If ingress port is ANY and reg_c[1] is mark_id,
4114  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4115  *
4116  * For default flow (zero mark_id), flow is like,
4117  *   - If ingress port is ANY,
4118  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4119  *
4120  * @param dev
4121  *   Pointer to Ethernet device.
4122  * @param mark_id
4123  *   ID of MARK action, zero means default flow for META.
4124  * @param[out] error
4125  *   Perform verbose error reporting if not NULL.
4126  *
4127  * @return
4128  *   Associated resource on success, NULL otherwise and rte_errno is set.
4129  */
4130 static struct mlx5_flow_mreg_copy_resource *
4131 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4132                           struct rte_flow_error *error)
4133 {
4134         struct mlx5_priv *priv = dev->data->dev_private;
4135         struct mlx5_hlist_entry *entry;
4136         struct mlx5_flow_cb_ctx ctx = {
4137                 .dev = dev,
4138                 .error = error,
4139         };
4140
4141         /* Check if already registered. */
4142         MLX5_ASSERT(priv->mreg_cp_tbl);
4143         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4144         if (!entry)
4145                 return NULL;
4146         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4147                             hlist_ent);
4148 }
4149
4150 void
4151 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
4152 {
4153         struct mlx5_flow_mreg_copy_resource *mcp_res =
4154                 container_of(entry, typeof(*mcp_res), hlist_ent);
4155         struct rte_eth_dev *dev = list->ctx;
4156         struct mlx5_priv *priv = dev->data->dev_private;
4157
4158         MLX5_ASSERT(mcp_res->rix_flow);
4159         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
4160         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4161 }
4162
4163 /**
4164  * Release flow in RX_CP_TBL.
4165  *
4166  * @param dev
4167  *   Pointer to Ethernet device.
4168  * @flow
4169  *   Parent flow for wich copying is provided.
4170  */
4171 static void
4172 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4173                           struct rte_flow *flow)
4174 {
4175         struct mlx5_flow_mreg_copy_resource *mcp_res;
4176         struct mlx5_priv *priv = dev->data->dev_private;
4177
4178         if (!flow->rix_mreg_copy)
4179                 return;
4180         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4181                                  flow->rix_mreg_copy);
4182         if (!mcp_res || !priv->mreg_cp_tbl)
4183                 return;
4184         MLX5_ASSERT(mcp_res->rix_flow);
4185         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4186         flow->rix_mreg_copy = 0;
4187 }
4188
4189 /**
4190  * Remove the default copy action from RX_CP_TBL.
4191  *
4192  * This functions is called in the mlx5_dev_start(). No thread safe
4193  * is guaranteed.
4194  *
4195  * @param dev
4196  *   Pointer to Ethernet device.
4197  */
4198 static void
4199 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4200 {
4201         struct mlx5_hlist_entry *entry;
4202         struct mlx5_priv *priv = dev->data->dev_private;
4203
4204         /* Check if default flow is registered. */
4205         if (!priv->mreg_cp_tbl)
4206                 return;
4207         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
4208                                   MLX5_DEFAULT_COPY_ID, NULL);
4209         if (!entry)
4210                 return;
4211         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4212 }
4213
4214 /**
4215  * Add the default copy action in in RX_CP_TBL.
4216  *
4217  * This functions is called in the mlx5_dev_start(). No thread safe
4218  * is guaranteed.
4219  *
4220  * @param dev
4221  *   Pointer to Ethernet device.
4222  * @param[out] error
4223  *   Perform verbose error reporting if not NULL.
4224  *
4225  * @return
4226  *   0 for success, negative value otherwise and rte_errno is set.
4227  */
4228 static int
4229 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4230                                   struct rte_flow_error *error)
4231 {
4232         struct mlx5_priv *priv = dev->data->dev_private;
4233         struct mlx5_flow_mreg_copy_resource *mcp_res;
4234
4235         /* Check whether extensive metadata feature is engaged. */
4236         if (!priv->config.dv_flow_en ||
4237             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4238             !mlx5_flow_ext_mreg_supported(dev) ||
4239             !priv->sh->dv_regc0_mask)
4240                 return 0;
4241         /*
4242          * Add default mreg copy flow may be called multiple time, but
4243          * only be called once in stop. Avoid register it twice.
4244          */
4245         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
4246                 return 0;
4247         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4248         if (!mcp_res)
4249                 return -rte_errno;
4250         return 0;
4251 }
4252
4253 /**
4254  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4255  *
4256  * All the flow having Q/RSS action should be split by
4257  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4258  * performs the following,
4259  *   - CQE->flow_tag := reg_c[1] (MARK)
4260  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4261  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4262  * but there should be a flow per each MARK ID set by MARK action.
4263  *
4264  * For the aforementioned reason, if there's a MARK action in flow's action
4265  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4266  * the MARK ID to CQE's flow_tag like,
4267  *   - If reg_c[1] is mark_id,
4268  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4269  *
4270  * For SET_META action which stores value in reg_c[0], as the destination is
4271  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4272  * MARK ID means the default flow. The default flow looks like,
4273  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4274  *
4275  * @param dev
4276  *   Pointer to Ethernet device.
4277  * @param flow
4278  *   Pointer to flow structure.
4279  * @param[in] actions
4280  *   Pointer to the list of actions.
4281  * @param[out] error
4282  *   Perform verbose error reporting if not NULL.
4283  *
4284  * @return
4285  *   0 on success, negative value otherwise and rte_errno is set.
4286  */
4287 static int
4288 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4289                             struct rte_flow *flow,
4290                             const struct rte_flow_action *actions,
4291                             struct rte_flow_error *error)
4292 {
4293         struct mlx5_priv *priv = dev->data->dev_private;
4294         struct mlx5_dev_config *config = &priv->config;
4295         struct mlx5_flow_mreg_copy_resource *mcp_res;
4296         const struct rte_flow_action_mark *mark;
4297
4298         /* Check whether extensive metadata feature is engaged. */
4299         if (!config->dv_flow_en ||
4300             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4301             !mlx5_flow_ext_mreg_supported(dev) ||
4302             !priv->sh->dv_regc0_mask)
4303                 return 0;
4304         /* Find MARK action. */
4305         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4306                 switch (actions->type) {
4307                 case RTE_FLOW_ACTION_TYPE_FLAG:
4308                         mcp_res = flow_mreg_add_copy_action
4309                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4310                         if (!mcp_res)
4311                                 return -rte_errno;
4312                         flow->rix_mreg_copy = mcp_res->idx;
4313                         return 0;
4314                 case RTE_FLOW_ACTION_TYPE_MARK:
4315                         mark = (const struct rte_flow_action_mark *)
4316                                 actions->conf;
4317                         mcp_res =
4318                                 flow_mreg_add_copy_action(dev, mark->id, error);
4319                         if (!mcp_res)
4320                                 return -rte_errno;
4321                         flow->rix_mreg_copy = mcp_res->idx;
4322                         return 0;
4323                 default:
4324                         break;
4325                 }
4326         }
4327         return 0;
4328 }
4329
4330 #define MLX5_MAX_SPLIT_ACTIONS 24
4331 #define MLX5_MAX_SPLIT_ITEMS 24
4332
4333 /**
4334  * Split the hairpin flow.
4335  * Since HW can't support encap and push-vlan on Rx, we move these
4336  * actions to Tx.
4337  * If the count action is after the encap then we also
4338  * move the count action. in this case the count will also measure
4339  * the outer bytes.
4340  *
4341  * @param dev
4342  *   Pointer to Ethernet device.
4343  * @param[in] actions
4344  *   Associated actions (list terminated by the END action).
4345  * @param[out] actions_rx
4346  *   Rx flow actions.
4347  * @param[out] actions_tx
4348  *   Tx flow actions..
4349  * @param[out] pattern_tx
4350  *   The pattern items for the Tx flow.
4351  * @param[out] flow_id
4352  *   The flow ID connected to this flow.
4353  *
4354  * @return
4355  *   0 on success.
4356  */
4357 static int
4358 flow_hairpin_split(struct rte_eth_dev *dev,
4359                    const struct rte_flow_action actions[],
4360                    struct rte_flow_action actions_rx[],
4361                    struct rte_flow_action actions_tx[],
4362                    struct rte_flow_item pattern_tx[],
4363                    uint32_t flow_id)
4364 {
4365         const struct rte_flow_action_raw_encap *raw_encap;
4366         const struct rte_flow_action_raw_decap *raw_decap;
4367         struct mlx5_rte_flow_action_set_tag *set_tag;
4368         struct rte_flow_action *tag_action;
4369         struct mlx5_rte_flow_item_tag *tag_item;
4370         struct rte_flow_item *item;
4371         char *addr;
4372         int encap = 0;
4373
4374         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4375                 switch (actions->type) {
4376                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4377                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4378                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4379                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4380                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4381                         rte_memcpy(actions_tx, actions,
4382                                sizeof(struct rte_flow_action));
4383                         actions_tx++;
4384                         break;
4385                 case RTE_FLOW_ACTION_TYPE_COUNT:
4386                         if (encap) {
4387                                 rte_memcpy(actions_tx, actions,
4388                                            sizeof(struct rte_flow_action));
4389                                 actions_tx++;
4390                         } else {
4391                                 rte_memcpy(actions_rx, actions,
4392                                            sizeof(struct rte_flow_action));
4393                                 actions_rx++;
4394                         }
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4397                         raw_encap = actions->conf;
4398                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4399                                 memcpy(actions_tx, actions,
4400                                        sizeof(struct rte_flow_action));
4401                                 actions_tx++;
4402                                 encap = 1;
4403                         } else {
4404                                 rte_memcpy(actions_rx, actions,
4405                                            sizeof(struct rte_flow_action));
4406                                 actions_rx++;
4407                         }
4408                         break;
4409                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4410                         raw_decap = actions->conf;
4411                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4412                                 memcpy(actions_tx, actions,
4413                                        sizeof(struct rte_flow_action));
4414                                 actions_tx++;
4415                         } else {
4416                                 rte_memcpy(actions_rx, actions,
4417                                            sizeof(struct rte_flow_action));
4418                                 actions_rx++;
4419                         }
4420                         break;
4421                 default:
4422                         rte_memcpy(actions_rx, actions,
4423                                    sizeof(struct rte_flow_action));
4424                         actions_rx++;
4425                         break;
4426                 }
4427         }
4428         /* Add set meta action and end action for the Rx flow. */
4429         tag_action = actions_rx;
4430         tag_action->type = (enum rte_flow_action_type)
4431                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4432         actions_rx++;
4433         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4434         actions_rx++;
4435         set_tag = (void *)actions_rx;
4436         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4437                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4438                 .data = flow_id,
4439         };
4440         MLX5_ASSERT(set_tag->id > REG_NON);
4441         tag_action->conf = set_tag;
4442         /* Create Tx item list. */
4443         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4444         addr = (void *)&pattern_tx[2];
4445         item = pattern_tx;
4446         item->type = (enum rte_flow_item_type)
4447                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4448         tag_item = (void *)addr;
4449         tag_item->data = flow_id;
4450         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4451         MLX5_ASSERT(set_tag->id > REG_NON);
4452         item->spec = tag_item;
4453         addr += sizeof(struct mlx5_rte_flow_item_tag);
4454         tag_item = (void *)addr;
4455         tag_item->data = UINT32_MAX;
4456         tag_item->id = UINT16_MAX;
4457         item->mask = tag_item;
4458         item->last = NULL;
4459         item++;
4460         item->type = RTE_FLOW_ITEM_TYPE_END;
4461         return 0;
4462 }
4463
4464 /**
4465  * The last stage of splitting chain, just creates the subflow
4466  * without any modification.
4467  *
4468  * @param[in] dev
4469  *   Pointer to Ethernet device.
4470  * @param[in] flow
4471  *   Parent flow structure pointer.
4472  * @param[in, out] sub_flow
4473  *   Pointer to return the created subflow, may be NULL.
4474  * @param[in] attr
4475  *   Flow rule attributes.
4476  * @param[in] items
4477  *   Pattern specification (list terminated by the END pattern item).
4478  * @param[in] actions
4479  *   Associated actions (list terminated by the END action).
4480  * @param[in] flow_split_info
4481  *   Pointer to flow split info structure.
4482  * @param[out] error
4483  *   Perform verbose error reporting if not NULL.
4484  * @return
4485  *   0 on success, negative value otherwise
4486  */
4487 static int
4488 flow_create_split_inner(struct rte_eth_dev *dev,
4489                         struct rte_flow *flow,
4490                         struct mlx5_flow **sub_flow,
4491                         const struct rte_flow_attr *attr,
4492                         const struct rte_flow_item items[],
4493                         const struct rte_flow_action actions[],
4494                         struct mlx5_flow_split_info *flow_split_info,
4495                         struct rte_flow_error *error)
4496 {
4497         struct mlx5_flow *dev_flow;
4498
4499         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4500                                     flow_split_info->flow_idx, error);
4501         if (!dev_flow)
4502                 return -rte_errno;
4503         dev_flow->flow = flow;
4504         dev_flow->external = flow_split_info->external;
4505         dev_flow->skip_scale = flow_split_info->skip_scale;
4506         /* Subflow object was created, we must include one in the list. */
4507         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4508                       dev_flow->handle, next);
4509         /*
4510          * If dev_flow is as one of the suffix flow, some actions in suffix
4511          * flow may need some user defined item layer flags, and pass the
4512          * Metadate rxq mark flag to suffix flow as well.
4513          */
4514         if (flow_split_info->prefix_layers)
4515                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4516         if (flow_split_info->prefix_mark)
4517                 dev_flow->handle->mark = 1;
4518         if (sub_flow)
4519                 *sub_flow = dev_flow;
4520 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4521         dev_flow->dv.table_id = flow_split_info->table_id;
4522 #endif
4523         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4524 }
4525
4526 /**
4527  * Get the sub policy of a meter.
4528  *
4529  * @param[in] dev
4530  *   Pointer to Ethernet device.
4531  * @param[in] flow
4532  *   Parent flow structure pointer.
4533  * @param[in] policy_id;
4534  *   Meter Policy id.
4535  * @param[in] attr
4536  *   Flow rule attributes.
4537  * @param[in] items
4538  *   Pattern specification (list terminated by the END pattern item).
4539  * @param[out] error
4540  *   Perform verbose error reporting if not NULL.
4541  *
4542  * @return
4543  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4544  */
4545 static struct mlx5_flow_meter_sub_policy *
4546 get_meter_sub_policy(struct rte_eth_dev *dev,
4547                      struct rte_flow *flow,
4548                      uint32_t policy_id,
4549                      const struct rte_flow_attr *attr,
4550                      const struct rte_flow_item items[],
4551                      struct rte_flow_error *error)
4552 {
4553         struct mlx5_flow_meter_policy *policy;
4554         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4555
4556         policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL);
4557         if (!policy) {
4558                 rte_flow_error_set(error, EINVAL,
4559                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4560                                    "Failed to find Meter Policy.");
4561                 goto exit;
4562         }
4563         if (policy->is_rss) {
4564                 struct mlx5_flow_workspace *wks =
4565                                 mlx5_flow_get_thread_workspace();
4566                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4567                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4568                 uint32_t i;
4569
4570                 MLX5_ASSERT(wks);
4571                 /**
4572                  * This is a tmp dev_flow,
4573                  * no need to register any matcher for it in translate.
4574                  */
4575                 wks->skip_matcher_reg = 1;
4576                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4577                         struct mlx5_flow dev_flow = {0};
4578                         struct mlx5_flow_handle dev_handle = { {0} };
4579                         const void *rss_act = policy->act_cnt[i].rss->conf;
4580                         struct rte_flow_action rss_actions[2] = {
4581                                 [0] = {
4582                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4583                                         .conf = rss_act
4584                                 },
4585                                 [1] = {
4586                                         .type = RTE_FLOW_ACTION_TYPE_END,
4587                                         .conf = NULL
4588                                 }
4589                         };
4590
4591                         dev_flow.handle = &dev_handle;
4592                         dev_flow.ingress = attr->ingress;
4593                         dev_flow.flow = flow;
4594                         dev_flow.external = 0;
4595 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4596                         dev_flow.dv.transfer = attr->transfer;
4597 #endif
4598                         /* Translate RSS action to get rss hash fields. */
4599                         if (flow_drv_translate(dev, &dev_flow, attr,
4600                                                 items, rss_actions, error))
4601                                 goto exit;
4602                         rss_desc_v[i] = wks->rss_desc;
4603                         rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4604                         rss_desc_v[i].hash_fields = dev_flow.hash_fields;
4605                         rss_desc_v[i].queue_num = rss_desc_v[i].hash_fields ?
4606                                                   rss_desc_v[i].queue_num : 1;
4607                         rss_desc[i] = &rss_desc_v[i];
4608                 }
4609                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4610                                                 flow, policy, rss_desc);
4611         } else {
4612                 enum mlx5_meter_domain mtr_domain =
4613                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4614                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4615                                         MLX5_MTR_DOMAIN_INGRESS;
4616                 sub_policy = policy->sub_policys[mtr_domain][0];
4617         }
4618         if (!sub_policy) {
4619                 rte_flow_error_set(error, EINVAL,
4620                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4621                         "Failed to get meter sub-policy.");
4622                 goto exit;
4623         }
4624 exit:
4625         return sub_policy;
4626 }
4627
4628 /**
4629  * Split the meter flow.
4630  *
4631  * As meter flow will split to three sub flow, other than meter
4632  * action, the other actions make sense to only meter accepts
4633  * the packet. If it need to be dropped, no other additional
4634  * actions should be take.
4635  *
4636  * One kind of special action which decapsulates the L3 tunnel
4637  * header will be in the prefix sub flow, as not to take the
4638  * L3 tunnel header into account.
4639  *
4640  * @param[in] dev
4641  *   Pointer to Ethernet device.
4642  * @param[in] flow
4643  *   Parent flow structure pointer.
4644  * @param[in] fm
4645  *   Pointer to flow meter structure.
4646  * @param[in] attr
4647  *   Flow rule attributes.
4648  * @param[in] items
4649  *   Pattern specification (list terminated by the END pattern item).
4650  * @param[out] sfx_items
4651  *   Suffix flow match items (list terminated by the END pattern item).
4652  * @param[in] actions
4653  *   Associated actions (list terminated by the END action).
4654  * @param[out] actions_sfx
4655  *   Suffix flow actions.
4656  * @param[out] actions_pre
4657  *   Prefix flow actions.
4658  * @param[out] mtr_flow_id
4659  *   Pointer to meter flow id.
4660  * @param[out] error
4661  *   Perform verbose error reporting if not NULL.
4662  *
4663  * @return
4664  *   0 on success, a negative errno value otherwise and rte_errno is set.
4665  */
4666 static int
4667 flow_meter_split_prep(struct rte_eth_dev *dev,
4668                       struct rte_flow *flow,
4669                       struct mlx5_flow_meter_info *fm,
4670                       const struct rte_flow_attr *attr,
4671                       const struct rte_flow_item items[],
4672                       struct rte_flow_item sfx_items[],
4673                       const struct rte_flow_action actions[],
4674                       struct rte_flow_action actions_sfx[],
4675                       struct rte_flow_action actions_pre[],
4676                       uint32_t *mtr_flow_id,
4677                       struct rte_flow_error *error)
4678 {
4679         struct mlx5_priv *priv = dev->data->dev_private;
4680         struct rte_flow_action *tag_action = NULL;
4681         struct rte_flow_item *tag_item;
4682         struct mlx5_rte_flow_action_set_tag *set_tag;
4683         const struct rte_flow_action_raw_encap *raw_encap;
4684         const struct rte_flow_action_raw_decap *raw_decap;
4685         struct mlx5_rte_flow_item_tag *tag_item_spec;
4686         struct mlx5_rte_flow_item_tag *tag_item_mask;
4687         uint32_t tag_id = 0;
4688         bool copy_vlan = false;
4689         struct rte_flow_action *hw_mtr_action;
4690         struct rte_flow_action *action_pre_head = NULL;
4691         bool mtr_first = priv->sh->meter_aso_en &&
4692                         (attr->egress ||
4693                         (attr->transfer && priv->representor_id != UINT16_MAX));
4694         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4695         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4696                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4697         uint32_t flow_id = 0;
4698         uint32_t flow_id_reversed = 0;
4699         uint8_t flow_id_bits = 0;
4700         int shift;
4701
4702         /* For ASO meter, meter must be before tag in TX direction. */
4703         if (mtr_first) {
4704                 action_pre_head = actions_pre++;
4705                 /* Leave space for tag action. */
4706                 tag_action = actions_pre++;
4707         }
4708         /* Prepare the actions for prefix and suffix flow. */
4709         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4710                 struct rte_flow_action *action_cur = NULL;
4711
4712                 switch (actions->type) {
4713                 case RTE_FLOW_ACTION_TYPE_METER:
4714                         if (mtr_first) {
4715                                 action_cur = action_pre_head;
4716                         } else {
4717                                 /* Leave space for tag action. */
4718                                 tag_action = actions_pre++;
4719                                 action_cur = actions_pre++;
4720                         }
4721                         break;
4722                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4723                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4724                         action_cur = actions_pre++;
4725                         break;
4726                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4727                         raw_encap = actions->conf;
4728                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4729                                 action_cur = actions_pre++;
4730                         break;
4731                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4732                         raw_decap = actions->conf;
4733                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4734                                 action_cur = actions_pre++;
4735                         break;
4736                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4737                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4738                         copy_vlan = true;
4739                         break;
4740                 default:
4741                         break;
4742                 }
4743                 if (!action_cur)
4744                         action_cur = (fm->def_policy) ?
4745                                         actions_sfx++ : actions_pre++;
4746                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4747         }
4748         /* Add end action to the actions. */
4749         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4750         if (priv->sh->meter_aso_en) {
4751                 /**
4752                  * For ASO meter, need to add an extra jump action explicitly,
4753                  * to jump from meter to policer table.
4754                  */
4755                 struct mlx5_flow_meter_sub_policy *sub_policy;
4756                 struct mlx5_flow_tbl_data_entry *tbl_data;
4757
4758                 if (!fm->def_policy) {
4759                         sub_policy = get_meter_sub_policy(dev, flow,
4760                                                           fm->policy_id, attr,
4761                                                           items, error);
4762                         if (!sub_policy)
4763                                 return -rte_errno;
4764                 } else {
4765                         enum mlx5_meter_domain mtr_domain =
4766                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4767                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4768                                         MLX5_MTR_DOMAIN_INGRESS;
4769
4770                         sub_policy =
4771                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
4772                 }
4773                 tbl_data = container_of(sub_policy->tbl_rsc,
4774                                         struct mlx5_flow_tbl_data_entry, tbl);
4775                 hw_mtr_action = actions_pre++;
4776                 hw_mtr_action->type = (enum rte_flow_action_type)
4777                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
4778                 hw_mtr_action->conf = tbl_data->jump.action;
4779         }
4780         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4781         actions_pre++;
4782         if (!tag_action)
4783                 return rte_flow_error_set(error, ENOMEM,
4784                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4785                                         "No tag action space.");
4786         if (!mtr_flow_id) {
4787                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
4788                 goto exit;
4789         }
4790         /* Only default-policy Meter creates mtr flow id. */
4791         if (fm->def_policy) {
4792                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
4793                 if (!tag_id)
4794                         return rte_flow_error_set(error, ENOMEM,
4795                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4796                                         "Failed to allocate meter flow id.");
4797                 flow_id = tag_id - 1;
4798                 flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id);
4799                 flow_id_bits = flow_id_bits ? flow_id_bits : 1;
4800                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
4801                     mtr_reg_bits) {
4802                         mlx5_ipool_free(fm->flow_ipool, tag_id);
4803                         return rte_flow_error_set(error, EINVAL,
4804                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4805                                         "Meter flow id exceeds max limit.");
4806                 }
4807                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
4808                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
4809         }
4810         /* Prepare the suffix subflow items. */
4811         tag_item = sfx_items++;
4812         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4813                 int item_type = items->type;
4814
4815                 switch (item_type) {
4816                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4817                         memcpy(sfx_items, items, sizeof(*sfx_items));
4818                         sfx_items++;
4819                         break;
4820                 case RTE_FLOW_ITEM_TYPE_VLAN:
4821                         if (copy_vlan) {
4822                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4823                                 /*
4824                                  * Convert to internal match item, it is used
4825                                  * for vlan push and set vid.
4826                                  */
4827                                 sfx_items->type = (enum rte_flow_item_type)
4828                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4829                                 sfx_items++;
4830                         }
4831                         break;
4832                 default:
4833                         break;
4834                 }
4835         }
4836         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4837         sfx_items++;
4838         /* Build tag actions and items for meter_id/meter flow_id. */
4839         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
4840         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4841         tag_item_mask = tag_item_spec + 1;
4842         /* Both flow_id and meter_id share the same register. */
4843         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4844                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
4845                                                             0, error),
4846                 .offset = mtr_id_offset,
4847                 .length = mtr_reg_bits,
4848                 .data = flow->meter,
4849         };
4850         /*
4851          * The color Reg bits used by flow_id are growing from
4852          * msb to lsb, so must do bit reverse for flow_id val in RegC.
4853          */
4854         for (shift = 0; shift < flow_id_bits; shift++)
4855                 flow_id_reversed = (flow_id_reversed << 1) |
4856                                 ((flow_id >> shift) & 0x1);
4857         set_tag->data |=
4858                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
4859         tag_item_spec->id = set_tag->id;
4860         tag_item_spec->data = set_tag->data << mtr_id_offset;
4861         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
4862         tag_action->type = (enum rte_flow_action_type)
4863                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4864         tag_action->conf = set_tag;
4865         tag_item->type = (enum rte_flow_item_type)
4866                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4867         tag_item->spec = tag_item_spec;
4868         tag_item->last = NULL;
4869         tag_item->mask = tag_item_mask;
4870 exit:
4871         if (mtr_flow_id)
4872                 *mtr_flow_id = tag_id;
4873         return 0;
4874 }
4875
4876 /**
4877  * Split action list having QUEUE/RSS for metadata register copy.
4878  *
4879  * Once Q/RSS action is detected in user's action list, the flow action
4880  * should be split in order to copy metadata registers, which will happen in
4881  * RX_CP_TBL like,
4882  *   - CQE->flow_tag := reg_c[1] (MARK)
4883  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4884  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4885  * This is because the last action of each flow must be a terminal action
4886  * (QUEUE, RSS or DROP).
4887  *
4888  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4889  * stored and kept in the mlx5_flow structure per each sub_flow.
4890  *
4891  * The Q/RSS action is replaced with,
4892  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4893  * And the following JUMP action is added at the end,
4894  *   - JUMP, to RX_CP_TBL.
4895  *
4896  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4897  * flow_create_split_metadata() routine. The flow will look like,
4898  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4899  *
4900  * @param dev
4901  *   Pointer to Ethernet device.
4902  * @param[out] split_actions
4903  *   Pointer to store split actions to jump to CP_TBL.
4904  * @param[in] actions
4905  *   Pointer to the list of original flow actions.
4906  * @param[in] qrss
4907  *   Pointer to the Q/RSS action.
4908  * @param[in] actions_n
4909  *   Number of original actions.
4910  * @param[out] error
4911  *   Perform verbose error reporting if not NULL.
4912  *
4913  * @return
4914  *   non-zero unique flow_id on success, otherwise 0 and
4915  *   error/rte_error are set.
4916  */
4917 static uint32_t
4918 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4919                           struct rte_flow_action *split_actions,
4920                           const struct rte_flow_action *actions,
4921                           const struct rte_flow_action *qrss,
4922                           int actions_n, struct rte_flow_error *error)
4923 {
4924         struct mlx5_priv *priv = dev->data->dev_private;
4925         struct mlx5_rte_flow_action_set_tag *set_tag;
4926         struct rte_flow_action_jump *jump;
4927         const int qrss_idx = qrss - actions;
4928         uint32_t flow_id = 0;
4929         int ret = 0;
4930
4931         /*
4932          * Given actions will be split
4933          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4934          * - Add jump to mreg CP_TBL.
4935          * As a result, there will be one more action.
4936          */
4937         ++actions_n;
4938         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4939         set_tag = (void *)(split_actions + actions_n);
4940         /*
4941          * If tag action is not set to void(it means we are not the meter
4942          * suffix flow), add the tag action. Since meter suffix flow already
4943          * has the tag added.
4944          */
4945         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4946                 /*
4947                  * Allocate the new subflow ID. This one is unique within
4948                  * device and not shared with representors. Otherwise,
4949                  * we would have to resolve multi-thread access synch
4950                  * issue. Each flow on the shared device is appended
4951                  * with source vport identifier, so the resulting
4952                  * flows will be unique in the shared (by master and
4953                  * representors) domain even if they have coinciding
4954                  * IDs.
4955                  */
4956                 mlx5_ipool_malloc(priv->sh->ipool
4957                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4958                 if (!flow_id)
4959                         return rte_flow_error_set(error, ENOMEM,
4960                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4961                                                   NULL, "can't allocate id "
4962                                                   "for split Q/RSS subflow");
4963                 /* Internal SET_TAG action to set flow ID. */
4964                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4965                         .data = flow_id,
4966                 };
4967                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4968                 if (ret < 0)
4969                         return ret;
4970                 set_tag->id = ret;
4971                 /* Construct new actions array. */
4972                 /* Replace QUEUE/RSS action. */
4973                 split_actions[qrss_idx] = (struct rte_flow_action){
4974                         .type = (enum rte_flow_action_type)
4975                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4976                         .conf = set_tag,
4977                 };
4978         }
4979         /* JUMP action to jump to mreg copy table (CP_TBL). */
4980         jump = (void *)(set_tag + 1);
4981         *jump = (struct rte_flow_action_jump){
4982                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4983         };
4984         split_actions[actions_n - 2] = (struct rte_flow_action){
4985                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4986                 .conf = jump,
4987         };
4988         split_actions[actions_n - 1] = (struct rte_flow_action){
4989                 .type = RTE_FLOW_ACTION_TYPE_END,
4990         };
4991         return flow_id;
4992 }
4993
4994 /**
4995  * Extend the given action list for Tx metadata copy.
4996  *
4997  * Copy the given action list to the ext_actions and add flow metadata register
4998  * copy action in order to copy reg_a set by WQE to reg_c[0].
4999  *
5000  * @param[out] ext_actions
5001  *   Pointer to the extended action list.
5002  * @param[in] actions
5003  *   Pointer to the list of actions.
5004  * @param[in] actions_n
5005  *   Number of actions in the list.
5006  * @param[out] error
5007  *   Perform verbose error reporting if not NULL.
5008  * @param[in] encap_idx
5009  *   The encap action inndex.
5010  *
5011  * @return
5012  *   0 on success, negative value otherwise
5013  */
5014 static int
5015 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5016                        struct rte_flow_action *ext_actions,
5017                        const struct rte_flow_action *actions,
5018                        int actions_n, struct rte_flow_error *error,
5019                        int encap_idx)
5020 {
5021         struct mlx5_flow_action_copy_mreg *cp_mreg =
5022                 (struct mlx5_flow_action_copy_mreg *)
5023                         (ext_actions + actions_n + 1);
5024         int ret;
5025
5026         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5027         if (ret < 0)
5028                 return ret;
5029         cp_mreg->dst = ret;
5030         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5031         if (ret < 0)
5032                 return ret;
5033         cp_mreg->src = ret;
5034         if (encap_idx != 0)
5035                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5036         if (encap_idx == actions_n - 1) {
5037                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5038                         .type = (enum rte_flow_action_type)
5039                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5040                         .conf = cp_mreg,
5041                 };
5042                 ext_actions[actions_n] = (struct rte_flow_action){
5043                         .type = RTE_FLOW_ACTION_TYPE_END,
5044                 };
5045         } else {
5046                 ext_actions[encap_idx] = (struct rte_flow_action){
5047                         .type = (enum rte_flow_action_type)
5048                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5049                         .conf = cp_mreg,
5050                 };
5051                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5052                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5053         }
5054         return 0;
5055 }
5056
5057 /**
5058  * Check the match action from the action list.
5059  *
5060  * @param[in] actions
5061  *   Pointer to the list of actions.
5062  * @param[in] attr
5063  *   Flow rule attributes.
5064  * @param[in] action
5065  *   The action to be check if exist.
5066  * @param[out] match_action_pos
5067  *   Pointer to the position of the matched action if exists, otherwise is -1.
5068  * @param[out] qrss_action_pos
5069  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5070  * @param[out] modify_after_mirror
5071  *   Pointer to the flag of modify action after FDB mirroring.
5072  *
5073  * @return
5074  *   > 0 the total number of actions.
5075  *   0 if not found match action in action list.
5076  */
5077 static int
5078 flow_check_match_action(const struct rte_flow_action actions[],
5079                         const struct rte_flow_attr *attr,
5080                         enum rte_flow_action_type action,
5081                         int *match_action_pos, int *qrss_action_pos,
5082                         int *modify_after_mirror)
5083 {
5084         const struct rte_flow_action_sample *sample;
5085         int actions_n = 0;
5086         uint32_t ratio = 0;
5087         int sub_type = 0;
5088         int flag = 0;
5089         int fdb_mirror = 0;
5090
5091         *match_action_pos = -1;
5092         *qrss_action_pos = -1;
5093         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5094                 if (actions->type == action) {
5095                         flag = 1;
5096                         *match_action_pos = actions_n;
5097                 }
5098                 switch (actions->type) {
5099                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5100                 case RTE_FLOW_ACTION_TYPE_RSS:
5101                         *qrss_action_pos = actions_n;
5102                         break;
5103                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5104                         sample = actions->conf;
5105                         ratio = sample->ratio;
5106                         sub_type = ((const struct rte_flow_action *)
5107                                         (sample->actions))->type;
5108                         if (ratio == 1 && attr->transfer)
5109                                 fdb_mirror = 1;
5110                         break;
5111                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5112                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5113                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5114                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5115                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5116                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5117                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5118                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5119                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5120                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5121                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5122                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5123                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5124                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5125                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5126                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5127                 case RTE_FLOW_ACTION_TYPE_FLAG:
5128                 case RTE_FLOW_ACTION_TYPE_MARK:
5129                 case RTE_FLOW_ACTION_TYPE_SET_META:
5130                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5131                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5132                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5133                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5134                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5135                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5136                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5137                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5138                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5139                 case RTE_FLOW_ACTION_TYPE_METER:
5140                         if (fdb_mirror)
5141                                 *modify_after_mirror = 1;
5142                         break;
5143                 default:
5144                         break;
5145                 }
5146                 actions_n++;
5147         }
5148         if (flag && fdb_mirror && !*modify_after_mirror) {
5149                 /* FDB mirroring uses the destination array to implement
5150                  * instead of FLOW_SAMPLER object.
5151                  */
5152                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5153                         flag = 0;
5154         }
5155         /* Count RTE_FLOW_ACTION_TYPE_END. */
5156         return flag ? actions_n + 1 : 0;
5157 }
5158
5159 #define SAMPLE_SUFFIX_ITEM 2
5160
5161 /**
5162  * Split the sample flow.
5163  *
5164  * As sample flow will split to two sub flow, sample flow with
5165  * sample action, the other actions will move to new suffix flow.
5166  *
5167  * Also add unique tag id with tag action in the sample flow,
5168  * the same tag id will be as match in the suffix flow.
5169  *
5170  * @param dev
5171  *   Pointer to Ethernet device.
5172  * @param[in] add_tag
5173  *   Add extra tag action flag.
5174  * @param[out] sfx_items
5175  *   Suffix flow match items (list terminated by the END pattern item).
5176  * @param[in] actions
5177  *   Associated actions (list terminated by the END action).
5178  * @param[out] actions_sfx
5179  *   Suffix flow actions.
5180  * @param[out] actions_pre
5181  *   Prefix flow actions.
5182  * @param[in] actions_n
5183  *  The total number of actions.
5184  * @param[in] sample_action_pos
5185  *   The sample action position.
5186  * @param[in] qrss_action_pos
5187  *   The Queue/RSS action position.
5188  * @param[in] jump_table
5189  *   Add extra jump action flag.
5190  * @param[out] error
5191  *   Perform verbose error reporting if not NULL.
5192  *
5193  * @return
5194  *   0 on success, or unique flow_id, a negative errno value
5195  *   otherwise and rte_errno is set.
5196  */
5197 static int
5198 flow_sample_split_prep(struct rte_eth_dev *dev,
5199                        int add_tag,
5200                        struct rte_flow_item sfx_items[],
5201                        const struct rte_flow_action actions[],
5202                        struct rte_flow_action actions_sfx[],
5203                        struct rte_flow_action actions_pre[],
5204                        int actions_n,
5205                        int sample_action_pos,
5206                        int qrss_action_pos,
5207                        int jump_table,
5208                        struct rte_flow_error *error)
5209 {
5210         struct mlx5_priv *priv = dev->data->dev_private;
5211         struct mlx5_rte_flow_action_set_tag *set_tag;
5212         struct mlx5_rte_flow_item_tag *tag_spec;
5213         struct mlx5_rte_flow_item_tag *tag_mask;
5214         struct rte_flow_action_jump *jump_action;
5215         uint32_t tag_id = 0;
5216         int index;
5217         int append_index = 0;
5218         int ret;
5219
5220         if (sample_action_pos < 0)
5221                 return rte_flow_error_set(error, EINVAL,
5222                                           RTE_FLOW_ERROR_TYPE_ACTION,
5223                                           NULL, "invalid position of sample "
5224                                           "action in list");
5225         /* Prepare the actions for prefix and suffix flow. */
5226         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5227                 index = qrss_action_pos;
5228                 /* Put the preceding the Queue/RSS action into prefix flow. */
5229                 if (index != 0)
5230                         memcpy(actions_pre, actions,
5231                                sizeof(struct rte_flow_action) * index);
5232                 /* Put others preceding the sample action into prefix flow. */
5233                 if (sample_action_pos > index + 1)
5234                         memcpy(actions_pre + index, actions + index + 1,
5235                                sizeof(struct rte_flow_action) *
5236                                (sample_action_pos - index - 1));
5237                 index = sample_action_pos - 1;
5238                 /* Put Queue/RSS action into Suffix flow. */
5239                 memcpy(actions_sfx, actions + qrss_action_pos,
5240                        sizeof(struct rte_flow_action));
5241                 actions_sfx++;
5242         } else {
5243                 index = sample_action_pos;
5244                 if (index != 0)
5245                         memcpy(actions_pre, actions,
5246                                sizeof(struct rte_flow_action) * index);
5247         }
5248         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5249          * For CX6DX and above, metadata registers Cx preserve their value,
5250          * add an extra tag action for NIC-RX and E-Switch Domain.
5251          */
5252         if (add_tag) {
5253                 /* Prepare the prefix tag action. */
5254                 append_index++;
5255                 set_tag = (void *)(actions_pre + actions_n + append_index);
5256                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5257                 if (ret < 0)
5258                         return ret;
5259                 mlx5_ipool_malloc(priv->sh->ipool
5260                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5261                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5262                         .id = ret,
5263                         .data = tag_id,
5264                 };
5265                 /* Prepare the suffix subflow items. */
5266                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5267                 tag_spec->data = tag_id;
5268                 tag_spec->id = set_tag->id;
5269                 tag_mask = tag_spec + 1;
5270                 tag_mask->data = UINT32_MAX;
5271                 sfx_items[0] = (struct rte_flow_item){
5272                         .type = (enum rte_flow_item_type)
5273                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5274                         .spec = tag_spec,
5275                         .last = NULL,
5276                         .mask = tag_mask,
5277                 };
5278                 sfx_items[1] = (struct rte_flow_item){
5279                         .type = (enum rte_flow_item_type)
5280                                 RTE_FLOW_ITEM_TYPE_END,
5281                 };
5282                 /* Prepare the tag action in prefix subflow. */
5283                 actions_pre[index++] =
5284                         (struct rte_flow_action){
5285                         .type = (enum rte_flow_action_type)
5286                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5287                         .conf = set_tag,
5288                 };
5289         }
5290         memcpy(actions_pre + index, actions + sample_action_pos,
5291                sizeof(struct rte_flow_action));
5292         index += 1;
5293         /* For the modify action after the sample action in E-Switch mirroring,
5294          * Add the extra jump action in prefix subflow and jump into the next
5295          * table, then do the modify action in the new table.
5296          */
5297         if (jump_table) {
5298                 /* Prepare the prefix jump action. */
5299                 append_index++;
5300                 jump_action = (void *)(actions_pre + actions_n + append_index);
5301                 jump_action->group = jump_table;
5302                 actions_pre[index++] =
5303                         (struct rte_flow_action){
5304                         .type = (enum rte_flow_action_type)
5305                                 RTE_FLOW_ACTION_TYPE_JUMP,
5306                         .conf = jump_action,
5307                 };
5308         }
5309         actions_pre[index] = (struct rte_flow_action){
5310                 .type = (enum rte_flow_action_type)
5311                         RTE_FLOW_ACTION_TYPE_END,
5312         };
5313         /* Put the actions after sample into Suffix flow. */
5314         memcpy(actions_sfx, actions + sample_action_pos + 1,
5315                sizeof(struct rte_flow_action) *
5316                (actions_n - sample_action_pos - 1));
5317         return tag_id;
5318 }
5319
5320 /**
5321  * The splitting for metadata feature.
5322  *
5323  * - Q/RSS action on NIC Rx should be split in order to pass by
5324  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5325  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5326  *
5327  * - All the actions on NIC Tx should have a mreg copy action to
5328  *   copy reg_a from WQE to reg_c[0].
5329  *
5330  * @param dev
5331  *   Pointer to Ethernet device.
5332  * @param[in] flow
5333  *   Parent flow structure pointer.
5334  * @param[in] attr
5335  *   Flow rule attributes.
5336  * @param[in] items
5337  *   Pattern specification (list terminated by the END pattern item).
5338  * @param[in] actions
5339  *   Associated actions (list terminated by the END action).
5340  * @param[in] flow_split_info
5341  *   Pointer to flow split info structure.
5342  * @param[out] error
5343  *   Perform verbose error reporting if not NULL.
5344  * @return
5345  *   0 on success, negative value otherwise
5346  */
5347 static int
5348 flow_create_split_metadata(struct rte_eth_dev *dev,
5349                            struct rte_flow *flow,
5350                            const struct rte_flow_attr *attr,
5351                            const struct rte_flow_item items[],
5352                            const struct rte_flow_action actions[],
5353                            struct mlx5_flow_split_info *flow_split_info,
5354                            struct rte_flow_error *error)
5355 {
5356         struct mlx5_priv *priv = dev->data->dev_private;
5357         struct mlx5_dev_config *config = &priv->config;
5358         const struct rte_flow_action *qrss = NULL;
5359         struct rte_flow_action *ext_actions = NULL;
5360         struct mlx5_flow *dev_flow = NULL;
5361         uint32_t qrss_id = 0;
5362         int mtr_sfx = 0;
5363         size_t act_size;
5364         int actions_n;
5365         int encap_idx;
5366         int ret;
5367
5368         /* Check whether extensive metadata feature is engaged. */
5369         if (!config->dv_flow_en ||
5370             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5371             !mlx5_flow_ext_mreg_supported(dev))
5372                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5373                                                actions, flow_split_info, error);
5374         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5375                                                            &encap_idx);
5376         if (qrss) {
5377                 /* Exclude hairpin flows from splitting. */
5378                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5379                         const struct rte_flow_action_queue *queue;
5380
5381                         queue = qrss->conf;
5382                         if (mlx5_rxq_get_type(dev, queue->index) ==
5383                             MLX5_RXQ_TYPE_HAIRPIN)
5384                                 qrss = NULL;
5385                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5386                         const struct rte_flow_action_rss *rss;
5387
5388                         rss = qrss->conf;
5389                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5390                             MLX5_RXQ_TYPE_HAIRPIN)
5391                                 qrss = NULL;
5392                 }
5393         }
5394         if (qrss) {
5395                 /* Check if it is in meter suffix table. */
5396                 mtr_sfx = attr->group == (attr->transfer ?
5397                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5398                           MLX5_FLOW_TABLE_LEVEL_METER);
5399                 /*
5400                  * Q/RSS action on NIC Rx should be split in order to pass by
5401                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5402                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5403                  */
5404                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5405                            sizeof(struct rte_flow_action_set_tag) +
5406                            sizeof(struct rte_flow_action_jump);
5407                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5408                                           SOCKET_ID_ANY);
5409                 if (!ext_actions)
5410                         return rte_flow_error_set(error, ENOMEM,
5411                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5412                                                   NULL, "no memory to split "
5413                                                   "metadata flow");
5414                 /*
5415                  * If we are the suffix flow of meter, tag already exist.
5416                  * Set the tag action to void.
5417                  */
5418                 if (mtr_sfx)
5419                         ext_actions[qrss - actions].type =
5420                                                 RTE_FLOW_ACTION_TYPE_VOID;
5421                 else
5422                         ext_actions[qrss - actions].type =
5423                                                 (enum rte_flow_action_type)
5424                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5425                 /*
5426                  * Create the new actions list with removed Q/RSS action
5427                  * and appended set tag and jump to register copy table
5428                  * (RX_CP_TBL). We should preallocate unique tag ID here
5429                  * in advance, because it is needed for set tag action.
5430                  */
5431                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5432                                                     qrss, actions_n, error);
5433                 if (!mtr_sfx && !qrss_id) {
5434                         ret = -rte_errno;
5435                         goto exit;
5436                 }
5437         } else if (attr->egress && !attr->transfer) {
5438                 /*
5439                  * All the actions on NIC Tx should have a metadata register
5440                  * copy action to copy reg_a from WQE to reg_c[meta]
5441                  */
5442                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5443                            sizeof(struct mlx5_flow_action_copy_mreg);
5444                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5445                                           SOCKET_ID_ANY);
5446                 if (!ext_actions)
5447                         return rte_flow_error_set(error, ENOMEM,
5448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5449                                                   NULL, "no memory to split "
5450                                                   "metadata flow");
5451                 /* Create the action list appended with copy register. */
5452                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5453                                              actions_n, error, encap_idx);
5454                 if (ret < 0)
5455                         goto exit;
5456         }
5457         /* Add the unmodified original or prefix subflow. */
5458         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5459                                       items, ext_actions ? ext_actions :
5460                                       actions, flow_split_info, error);
5461         if (ret < 0)
5462                 goto exit;
5463         MLX5_ASSERT(dev_flow);
5464         if (qrss) {
5465                 const struct rte_flow_attr q_attr = {
5466                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5467                         .ingress = 1,
5468                 };
5469                 /* Internal PMD action to set register. */
5470                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5471                         .data = qrss_id,
5472                         .id = REG_NON,
5473                 };
5474                 struct rte_flow_item q_items[] = {
5475                         {
5476                                 .type = (enum rte_flow_item_type)
5477                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5478                                 .spec = &q_tag_spec,
5479                                 .last = NULL,
5480                                 .mask = NULL,
5481                         },
5482                         {
5483                                 .type = RTE_FLOW_ITEM_TYPE_END,
5484                         },
5485                 };
5486                 struct rte_flow_action q_actions[] = {
5487                         {
5488                                 .type = qrss->type,
5489                                 .conf = qrss->conf,
5490                         },
5491                         {
5492                                 .type = RTE_FLOW_ACTION_TYPE_END,
5493                         },
5494                 };
5495                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5496
5497                 /*
5498                  * Configure the tag item only if there is no meter subflow.
5499                  * Since tag is already marked in the meter suffix subflow
5500                  * we can just use the meter suffix items as is.
5501                  */
5502                 if (qrss_id) {
5503                         /* Not meter subflow. */
5504                         MLX5_ASSERT(!mtr_sfx);
5505                         /*
5506                          * Put unique id in prefix flow due to it is destroyed
5507                          * after suffix flow and id will be freed after there
5508                          * is no actual flows with this id and identifier
5509                          * reallocation becomes possible (for example, for
5510                          * other flows in other threads).
5511                          */
5512                         dev_flow->handle->split_flow_id = qrss_id;
5513                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5514                                                    error);
5515                         if (ret < 0)
5516                                 goto exit;
5517                         q_tag_spec.id = ret;
5518                 }
5519                 dev_flow = NULL;
5520                 /* Add suffix subflow to execute Q/RSS. */
5521                 flow_split_info->prefix_layers = layers;
5522                 flow_split_info->prefix_mark = 0;
5523                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5524                                               &q_attr, mtr_sfx ? items :
5525                                               q_items, q_actions,
5526                                               flow_split_info, error);
5527                 if (ret < 0)
5528                         goto exit;
5529                 /* qrss ID should be freed if failed. */
5530                 qrss_id = 0;
5531                 MLX5_ASSERT(dev_flow);
5532         }
5533
5534 exit:
5535         /*
5536          * We do not destroy the partially created sub_flows in case of error.
5537          * These ones are included into parent flow list and will be destroyed
5538          * by flow_drv_destroy.
5539          */
5540         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5541                         qrss_id);
5542         mlx5_free(ext_actions);
5543         return ret;
5544 }
5545
5546 /**
5547  * Create meter internal drop flow with the original pattern.
5548  *
5549  * @param dev
5550  *   Pointer to Ethernet device.
5551  * @param[in] flow
5552  *   Parent flow structure pointer.
5553  * @param[in] attr
5554  *   Flow rule attributes.
5555  * @param[in] items
5556  *   Pattern specification (list terminated by the END pattern item).
5557  * @param[in] flow_split_info
5558  *   Pointer to flow split info structure.
5559  * @param[in] fm
5560  *   Pointer to flow meter structure.
5561  * @param[out] error
5562  *   Perform verbose error reporting if not NULL.
5563  * @return
5564  *   0 on success, negative value otherwise
5565  */
5566 static uint32_t
5567 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5568                         struct rte_flow *flow,
5569                         const struct rte_flow_attr *attr,
5570                         const struct rte_flow_item items[],
5571                         struct mlx5_flow_split_info *flow_split_info,
5572                         struct mlx5_flow_meter_info *fm,
5573                         struct rte_flow_error *error)
5574 {
5575         struct mlx5_flow *dev_flow = NULL;
5576         struct rte_flow_attr drop_attr = *attr;
5577         struct rte_flow_action drop_actions[3];
5578         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5579
5580         MLX5_ASSERT(fm->drop_cnt);
5581         drop_actions[0].type =
5582                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5583         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5584         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5585         drop_actions[1].conf = NULL;
5586         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5587         drop_actions[2].conf = NULL;
5588         drop_split_info.external = false;
5589         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5590         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5591         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5592         return flow_create_split_inner(dev, flow, &dev_flow,
5593                                 &drop_attr, items, drop_actions,
5594                                 &drop_split_info, error);
5595 }
5596
5597 /**
5598  * The splitting for meter feature.
5599  *
5600  * - The meter flow will be split to two flows as prefix and
5601  *   suffix flow. The packets make sense only it pass the prefix
5602  *   meter action.
5603  *
5604  * - Reg_C_5 is used for the packet to match betweend prefix and
5605  *   suffix flow.
5606  *
5607  * @param dev
5608  *   Pointer to Ethernet device.
5609  * @param[in] flow
5610  *   Parent flow structure pointer.
5611  * @param[in] attr
5612  *   Flow rule attributes.
5613  * @param[in] items
5614  *   Pattern specification (list terminated by the END pattern item).
5615  * @param[in] actions
5616  *   Associated actions (list terminated by the END action).
5617  * @param[in] flow_split_info
5618  *   Pointer to flow split info structure.
5619  * @param[out] error
5620  *   Perform verbose error reporting if not NULL.
5621  * @return
5622  *   0 on success, negative value otherwise
5623  */
5624 static int
5625 flow_create_split_meter(struct rte_eth_dev *dev,
5626                         struct rte_flow *flow,
5627                         const struct rte_flow_attr *attr,
5628                         const struct rte_flow_item items[],
5629                         const struct rte_flow_action actions[],
5630                         struct mlx5_flow_split_info *flow_split_info,
5631                         struct rte_flow_error *error)
5632 {
5633         struct mlx5_priv *priv = dev->data->dev_private;
5634         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5635         struct rte_flow_action *sfx_actions = NULL;
5636         struct rte_flow_action *pre_actions = NULL;
5637         struct rte_flow_item *sfx_items = NULL;
5638         struct mlx5_flow *dev_flow = NULL;
5639         struct rte_flow_attr sfx_attr = *attr;
5640         struct mlx5_flow_meter_info *fm = NULL;
5641         uint8_t skip_scale_restore;
5642         bool has_mtr = false;
5643         bool has_modify = false;
5644         bool set_mtr_reg = true;
5645         uint32_t meter_id = 0;
5646         uint32_t mtr_idx = 0;
5647         uint32_t mtr_flow_id = 0;
5648         size_t act_size;
5649         size_t item_size;
5650         int actions_n = 0;
5651         int ret = 0;
5652
5653         if (priv->mtr_en)
5654                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5655                                                     &has_modify, &meter_id);
5656         if (has_mtr) {
5657                 if (flow->meter) {
5658                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5659                         if (!fm)
5660                                 return rte_flow_error_set(error, EINVAL,
5661                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5662                                                 NULL, "Meter not found.");
5663                 } else {
5664                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5665                         if (!fm)
5666                                 return rte_flow_error_set(error, EINVAL,
5667                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5668                                                 NULL, "Meter not found.");
5669                         ret = mlx5_flow_meter_attach(priv, fm,
5670                                                      &sfx_attr, error);
5671                         if (ret)
5672                                 return -rte_errno;
5673                         flow->meter = mtr_idx;
5674                 }
5675                 MLX5_ASSERT(wks);
5676                 wks->fm = fm;
5677                 /*
5678                  * If it isn't default-policy Meter, and
5679                  * 1. There's no action in flow to change
5680                  *    packet (modify/encap/decap etc.), OR
5681                  * 2. No drop count needed for this meter.
5682                  * no need to use regC to save meter id anymore.
5683                  */
5684                 if (!fm->def_policy && (!has_modify || !fm->drop_cnt))
5685                         set_mtr_reg = false;
5686                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5687                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5688                            sizeof(struct mlx5_rte_flow_action_set_tag);
5689                 /* Suffix items: tag, vlan, port id, end. */
5690 #define METER_SUFFIX_ITEM 4
5691                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5692                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5693                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5694                                           0, SOCKET_ID_ANY);
5695                 if (!sfx_actions)
5696                         return rte_flow_error_set(error, ENOMEM,
5697                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5698                                                   NULL, "no memory to split "
5699                                                   "meter flow");
5700                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5701                              act_size);
5702                 /* There's no suffix flow for meter of non-default policy. */
5703                 if (!fm->def_policy)
5704                         pre_actions = sfx_actions + 1;
5705                 else
5706                         pre_actions = sfx_actions + actions_n;
5707                 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr,
5708                                             items, sfx_items, actions,
5709                                             sfx_actions, pre_actions,
5710                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5711                                             error);
5712                 if (ret) {
5713                         ret = -rte_errno;
5714                         goto exit;
5715                 }
5716                 /* Add the prefix subflow. */
5717                 flow_split_info->prefix_mark = 0;
5718                 skip_scale_restore = flow_split_info->skip_scale;
5719                 flow_split_info->skip_scale |=
5720                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5721                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5722                                               attr, items, pre_actions,
5723                                               flow_split_info, error);
5724                 flow_split_info->skip_scale = skip_scale_restore;
5725                 if (ret) {
5726                         if (mtr_flow_id)
5727                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5728                         ret = -rte_errno;
5729                         goto exit;
5730                 }
5731                 if (mtr_flow_id) {
5732                         dev_flow->handle->split_flow_id = mtr_flow_id;
5733                         dev_flow->handle->is_meter_flow_id = 1;
5734                 }
5735                 if (!fm->def_policy) {
5736                         if (!set_mtr_reg && fm->drop_cnt)
5737                                 ret =
5738                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5739                                                         &sfx_attr, items,
5740                                                         flow_split_info,
5741                                                         fm, error);
5742                         goto exit;
5743                 }
5744                 /* Setting the sfx group atrr. */
5745                 sfx_attr.group = sfx_attr.transfer ?
5746                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5747                                  MLX5_FLOW_TABLE_LEVEL_METER;
5748                 flow_split_info->prefix_layers =
5749                                 flow_get_prefix_layer_flags(dev_flow);
5750                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5751                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5752         }
5753         /* Add the prefix subflow. */
5754         ret = flow_create_split_metadata(dev, flow,
5755                                          &sfx_attr, sfx_items ?
5756                                          sfx_items : items,
5757                                          sfx_actions ? sfx_actions : actions,
5758                                          flow_split_info, error);
5759 exit:
5760         if (sfx_actions)
5761                 mlx5_free(sfx_actions);
5762         return ret;
5763 }
5764
5765 /**
5766  * The splitting for sample feature.
5767  *
5768  * Once Sample action is detected in the action list, the flow actions should
5769  * be split into prefix sub flow and suffix sub flow.
5770  *
5771  * The original items remain in the prefix sub flow, all actions preceding the
5772  * sample action and the sample action itself will be copied to the prefix
5773  * sub flow, the actions following the sample action will be copied to the
5774  * suffix sub flow, Queue action always be located in the suffix sub flow.
5775  *
5776  * In order to make the packet from prefix sub flow matches with suffix sub
5777  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5778  * flow uses tag item with the unique flow id.
5779  *
5780  * @param dev
5781  *   Pointer to Ethernet device.
5782  * @param[in] flow
5783  *   Parent flow structure pointer.
5784  * @param[in] attr
5785  *   Flow rule attributes.
5786  * @param[in] items
5787  *   Pattern specification (list terminated by the END pattern item).
5788  * @param[in] actions
5789  *   Associated actions (list terminated by the END action).
5790  * @param[in] flow_split_info
5791  *   Pointer to flow split info structure.
5792  * @param[out] error
5793  *   Perform verbose error reporting if not NULL.
5794  * @return
5795  *   0 on success, negative value otherwise
5796  */
5797 static int
5798 flow_create_split_sample(struct rte_eth_dev *dev,
5799                          struct rte_flow *flow,
5800                          const struct rte_flow_attr *attr,
5801                          const struct rte_flow_item items[],
5802                          const struct rte_flow_action actions[],
5803                          struct mlx5_flow_split_info *flow_split_info,
5804                          struct rte_flow_error *error)
5805 {
5806         struct mlx5_priv *priv = dev->data->dev_private;
5807         struct rte_flow_action *sfx_actions = NULL;
5808         struct rte_flow_action *pre_actions = NULL;
5809         struct rte_flow_item *sfx_items = NULL;
5810         struct mlx5_flow *dev_flow = NULL;
5811         struct rte_flow_attr sfx_attr = *attr;
5812 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5813         struct mlx5_flow_dv_sample_resource *sample_res;
5814         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5815         struct mlx5_flow_tbl_resource *sfx_tbl;
5816 #endif
5817         size_t act_size;
5818         size_t item_size;
5819         uint32_t fdb_tx = 0;
5820         int32_t tag_id = 0;
5821         int actions_n = 0;
5822         int sample_action_pos;
5823         int qrss_action_pos;
5824         int add_tag = 0;
5825         int modify_after_mirror = 0;
5826         uint16_t jump_table = 0;
5827         const uint32_t next_ft_step = 1;
5828         int ret = 0;
5829
5830         if (priv->sampler_en)
5831                 actions_n = flow_check_match_action(actions, attr,
5832                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5833                                         &sample_action_pos, &qrss_action_pos,
5834                                         &modify_after_mirror);
5835         if (actions_n) {
5836                 /* The prefix actions must includes sample, tag, end. */
5837                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5838                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5839                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5840                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5841                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5842                                           item_size), 0, SOCKET_ID_ANY);
5843                 if (!sfx_actions)
5844                         return rte_flow_error_set(error, ENOMEM,
5845                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5846                                                   NULL, "no memory to split "
5847                                                   "sample flow");
5848                 /* The representor_id is -1 for uplink. */
5849                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5850                 /*
5851                  * When reg_c_preserve is set, metadata registers Cx preserve
5852                  * their value even through packet duplication.
5853                  */
5854                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
5855                 if (add_tag)
5856                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5857                                         + act_size);
5858                 if (modify_after_mirror)
5859                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
5860                                      next_ft_step;
5861                 pre_actions = sfx_actions + actions_n;
5862                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
5863                                                 actions, sfx_actions,
5864                                                 pre_actions, actions_n,
5865                                                 sample_action_pos,
5866                                                 qrss_action_pos, jump_table,
5867                                                 error);
5868                 if (tag_id < 0 || (add_tag && !tag_id)) {
5869                         ret = -rte_errno;
5870                         goto exit;
5871                 }
5872                 if (modify_after_mirror)
5873                         flow_split_info->skip_scale =
5874                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5875                 /* Add the prefix subflow. */
5876                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5877                                               items, pre_actions,
5878                                               flow_split_info, error);
5879                 if (ret) {
5880                         ret = -rte_errno;
5881                         goto exit;
5882                 }
5883                 dev_flow->handle->split_flow_id = tag_id;
5884 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5885                 if (!modify_after_mirror) {
5886                         /* Set the sfx group attr. */
5887                         sample_res = (struct mlx5_flow_dv_sample_resource *)
5888                                                 dev_flow->dv.sample_res;
5889                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
5890                                                 sample_res->normal_path_tbl;
5891                         sfx_tbl_data = container_of(sfx_tbl,
5892                                                 struct mlx5_flow_tbl_data_entry,
5893                                                 tbl);
5894                         sfx_attr.group = sfx_attr.transfer ?
5895                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
5896                 } else {
5897                         MLX5_ASSERT(attr->transfer);
5898                         sfx_attr.group = jump_table;
5899                 }
5900                 flow_split_info->prefix_layers =
5901                                 flow_get_prefix_layer_flags(dev_flow);
5902                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5903                 /* Suffix group level already be scaled with factor, set
5904                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
5905                  * again in translation.
5906                  */
5907                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5908 #endif
5909         }
5910         /* Add the suffix subflow. */
5911         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5912                                       sfx_items ? sfx_items : items,
5913                                       sfx_actions ? sfx_actions : actions,
5914                                       flow_split_info, error);
5915 exit:
5916         if (sfx_actions)
5917                 mlx5_free(sfx_actions);
5918         return ret;
5919 }
5920
5921 /**
5922  * Split the flow to subflow set. The splitters might be linked
5923  * in the chain, like this:
5924  * flow_create_split_outer() calls:
5925  *   flow_create_split_meter() calls:
5926  *     flow_create_split_metadata(meter_subflow_0) calls:
5927  *       flow_create_split_inner(metadata_subflow_0)
5928  *       flow_create_split_inner(metadata_subflow_1)
5929  *       flow_create_split_inner(metadata_subflow_2)
5930  *     flow_create_split_metadata(meter_subflow_1) calls:
5931  *       flow_create_split_inner(metadata_subflow_0)
5932  *       flow_create_split_inner(metadata_subflow_1)
5933  *       flow_create_split_inner(metadata_subflow_2)
5934  *
5935  * This provide flexible way to add new levels of flow splitting.
5936  * The all of successfully created subflows are included to the
5937  * parent flow dev_flow list.
5938  *
5939  * @param dev
5940  *   Pointer to Ethernet device.
5941  * @param[in] flow
5942  *   Parent flow structure pointer.
5943  * @param[in] attr
5944  *   Flow rule attributes.
5945  * @param[in] items
5946  *   Pattern specification (list terminated by the END pattern item).
5947  * @param[in] actions
5948  *   Associated actions (list terminated by the END action).
5949  * @param[in] flow_split_info
5950  *   Pointer to flow split info structure.
5951  * @param[out] error
5952  *   Perform verbose error reporting if not NULL.
5953  * @return
5954  *   0 on success, negative value otherwise
5955  */
5956 static int
5957 flow_create_split_outer(struct rte_eth_dev *dev,
5958                         struct rte_flow *flow,
5959                         const struct rte_flow_attr *attr,
5960                         const struct rte_flow_item items[],
5961                         const struct rte_flow_action actions[],
5962                         struct mlx5_flow_split_info *flow_split_info,
5963                         struct rte_flow_error *error)
5964 {
5965         int ret;
5966
5967         ret = flow_create_split_sample(dev, flow, attr, items,
5968                                        actions, flow_split_info, error);
5969         MLX5_ASSERT(ret <= 0);
5970         return ret;
5971 }
5972
5973 static inline struct mlx5_flow_tunnel *
5974 flow_tunnel_from_rule(const struct mlx5_flow *flow)
5975 {
5976         struct mlx5_flow_tunnel *tunnel;
5977
5978 #pragma GCC diagnostic push
5979 #pragma GCC diagnostic ignored "-Wcast-qual"
5980         tunnel = (typeof(tunnel))flow->tunnel;
5981 #pragma GCC diagnostic pop
5982
5983         return tunnel;
5984 }
5985
5986 /**
5987  * Adjust flow RSS workspace if needed.
5988  *
5989  * @param wks
5990  *   Pointer to thread flow work space.
5991  * @param rss_desc
5992  *   Pointer to RSS descriptor.
5993  * @param[in] nrssq_num
5994  *   New RSS queue number.
5995  *
5996  * @return
5997  *   0 on success, -1 otherwise and rte_errno is set.
5998  */
5999 static int
6000 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6001                           struct mlx5_flow_rss_desc *rss_desc,
6002                           uint32_t nrssq_num)
6003 {
6004         if (likely(nrssq_num <= wks->rssq_num))
6005                 return 0;
6006         rss_desc->queue = realloc(rss_desc->queue,
6007                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6008         if (!rss_desc->queue) {
6009                 rte_errno = ENOMEM;
6010                 return -1;
6011         }
6012         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6013         return 0;
6014 }
6015
6016 /**
6017  * Create a flow and add it to @p list.
6018  *
6019  * @param dev
6020  *   Pointer to Ethernet device.
6021  * @param list
6022  *   Pointer to a TAILQ flow list. If this parameter NULL,
6023  *   no list insertion occurred, flow is just created,
6024  *   this is caller's responsibility to track the
6025  *   created flow.
6026  * @param[in] attr
6027  *   Flow rule attributes.
6028  * @param[in] items
6029  *   Pattern specification (list terminated by the END pattern item).
6030  * @param[in] actions
6031  *   Associated actions (list terminated by the END action).
6032  * @param[in] external
6033  *   This flow rule is created by request external to PMD.
6034  * @param[out] error
6035  *   Perform verbose error reporting if not NULL.
6036  *
6037  * @return
6038  *   A flow index on success, 0 otherwise and rte_errno is set.
6039  */
6040 static uint32_t
6041 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
6042                  const struct rte_flow_attr *attr,
6043                  const struct rte_flow_item items[],
6044                  const struct rte_flow_action original_actions[],
6045                  bool external, struct rte_flow_error *error)
6046 {
6047         struct mlx5_priv *priv = dev->data->dev_private;
6048         struct rte_flow *flow = NULL;
6049         struct mlx5_flow *dev_flow;
6050         const struct rte_flow_action_rss *rss = NULL;
6051         struct mlx5_translated_action_handle
6052                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6053         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6054         union {
6055                 struct mlx5_flow_expand_rss buf;
6056                 uint8_t buffer[2048];
6057         } expand_buffer;
6058         union {
6059                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6060                 uint8_t buffer[2048];
6061         } actions_rx;
6062         union {
6063                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6064                 uint8_t buffer[2048];
6065         } actions_hairpin_tx;
6066         union {
6067                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6068                 uint8_t buffer[2048];
6069         } items_tx;
6070         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6071         struct mlx5_flow_rss_desc *rss_desc;
6072         const struct rte_flow_action *p_actions_rx;
6073         uint32_t i;
6074         uint32_t idx = 0;
6075         int hairpin_flow;
6076         struct rte_flow_attr attr_tx = { .priority = 0 };
6077         const struct rte_flow_action *actions;
6078         struct rte_flow_action *translated_actions = NULL;
6079         struct mlx5_flow_tunnel *tunnel;
6080         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6081         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6082         struct mlx5_flow_split_info flow_split_info = {
6083                 .external = !!external,
6084                 .skip_scale = 0,
6085                 .flow_idx = 0,
6086                 .prefix_mark = 0,
6087                 .prefix_layers = 0,
6088                 .table_id = 0
6089         };
6090         int ret;
6091
6092         MLX5_ASSERT(wks);
6093         rss_desc = &wks->rss_desc;
6094         ret = flow_action_handles_translate(dev, original_actions,
6095                                             indir_actions,
6096                                             &indir_actions_n,
6097                                             &translated_actions, error);
6098         if (ret < 0) {
6099                 MLX5_ASSERT(translated_actions == NULL);
6100                 return 0;
6101         }
6102         actions = translated_actions ? translated_actions : original_actions;
6103         p_actions_rx = actions;
6104         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6105         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6106                                 external, hairpin_flow, error);
6107         if (ret < 0)
6108                 goto error_before_hairpin_split;
6109         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
6110         if (!flow) {
6111                 rte_errno = ENOMEM;
6112                 goto error_before_hairpin_split;
6113         }
6114         if (hairpin_flow > 0) {
6115                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6116                         rte_errno = EINVAL;
6117                         goto error_before_hairpin_split;
6118                 }
6119                 flow_hairpin_split(dev, actions, actions_rx.actions,
6120                                    actions_hairpin_tx.actions, items_tx.items,
6121                                    idx);
6122                 p_actions_rx = actions_rx.actions;
6123         }
6124         flow_split_info.flow_idx = idx;
6125         flow->drv_type = flow_get_drv_type(dev, attr);
6126         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6127                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6128         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6129         /* RSS Action only works on NIC RX domain */
6130         if (attr->ingress && !attr->transfer)
6131                 rss = flow_get_rss_action(dev, p_actions_rx);
6132         if (rss) {
6133                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6134                         return 0;
6135                 /*
6136                  * The following information is required by
6137                  * mlx5_flow_hashfields_adjust() in advance.
6138                  */
6139                 rss_desc->level = rss->level;
6140                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6141                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6142         }
6143         flow->dev_handles = 0;
6144         if (rss && rss->types) {
6145                 unsigned int graph_root;
6146
6147                 graph_root = find_graph_root(items, rss->level);
6148                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6149                                            items, rss->types,
6150                                            mlx5_support_expansion, graph_root);
6151                 MLX5_ASSERT(ret > 0 &&
6152                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6153         } else {
6154                 buf->entries = 1;
6155                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6156         }
6157         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6158                                                       indir_actions_n);
6159         for (i = 0; i < buf->entries; ++i) {
6160                 /* Initialize flow split data. */
6161                 flow_split_info.prefix_layers = 0;
6162                 flow_split_info.prefix_mark = 0;
6163                 flow_split_info.skip_scale = 0;
6164                 /*
6165                  * The splitter may create multiple dev_flows,
6166                  * depending on configuration. In the simplest
6167                  * case it just creates unmodified original flow.
6168                  */
6169                 ret = flow_create_split_outer(dev, flow, attr,
6170                                               buf->entry[i].pattern,
6171                                               p_actions_rx, &flow_split_info,
6172                                               error);
6173                 if (ret < 0)
6174                         goto error;
6175                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6176                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6177                                                            p_actions_rx,
6178                                                            idx,
6179                                                            wks->flows[0].tunnel,
6180                                                            &default_miss_ctx,
6181                                                            error);
6182                         if (ret < 0) {
6183                                 mlx5_free(default_miss_ctx.queue);
6184                                 goto error;
6185                         }
6186                 }
6187         }
6188         /* Create the tx flow. */
6189         if (hairpin_flow) {
6190                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6191                 attr_tx.ingress = 0;
6192                 attr_tx.egress = 1;
6193                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6194                                          actions_hairpin_tx.actions,
6195                                          idx, error);
6196                 if (!dev_flow)
6197                         goto error;
6198                 dev_flow->flow = flow;
6199                 dev_flow->external = 0;
6200                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6201                               dev_flow->handle, next);
6202                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6203                                          items_tx.items,
6204                                          actions_hairpin_tx.actions, error);
6205                 if (ret < 0)
6206                         goto error;
6207         }
6208         /*
6209          * Update the metadata register copy table. If extensive
6210          * metadata feature is enabled and registers are supported
6211          * we might create the extra rte_flow for each unique
6212          * MARK/FLAG action ID.
6213          *
6214          * The table is updated for ingress Flows only, because
6215          * the egress Flows belong to the different device and
6216          * copy table should be updated in peer NIC Rx domain.
6217          */
6218         if (attr->ingress &&
6219             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6220                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6221                 if (ret)
6222                         goto error;
6223         }
6224         /*
6225          * If the flow is external (from application) OR device is started,
6226          * OR mreg discover, then apply immediately.
6227          */
6228         if (external || dev->data->dev_started ||
6229             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6230              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6231                 ret = flow_drv_apply(dev, flow, error);
6232                 if (ret < 0)
6233                         goto error;
6234         }
6235         if (list) {
6236                 rte_spinlock_lock(&priv->flow_list_lock);
6237                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
6238                              flow, next);
6239                 rte_spinlock_unlock(&priv->flow_list_lock);
6240         }
6241         flow_rxq_flags_set(dev, flow);
6242         rte_free(translated_actions);
6243         tunnel = flow_tunnel_from_rule(wks->flows);
6244         if (tunnel) {
6245                 flow->tunnel = 1;
6246                 flow->tunnel_id = tunnel->tunnel_id;
6247                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6248                 mlx5_free(default_miss_ctx.queue);
6249         }
6250         mlx5_flow_pop_thread_workspace();
6251         return idx;
6252 error:
6253         MLX5_ASSERT(flow);
6254         ret = rte_errno; /* Save rte_errno before cleanup. */
6255         flow_mreg_del_copy_action(dev, flow);
6256         flow_drv_destroy(dev, flow);
6257         if (rss_desc->shared_rss)
6258                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6259                         mlx5_ipool_get
6260                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6261                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6262         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
6263         rte_errno = ret; /* Restore rte_errno. */
6264         ret = rte_errno;
6265         rte_errno = ret;
6266         mlx5_flow_pop_thread_workspace();
6267 error_before_hairpin_split:
6268         rte_free(translated_actions);
6269         return 0;
6270 }
6271
6272 /**
6273  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6274  * incoming packets to table 1.
6275  *
6276  * Other flow rules, requested for group n, will be created in
6277  * e-switch table n+1.
6278  * Jump action to e-switch group n will be created to group n+1.
6279  *
6280  * Used when working in switchdev mode, to utilise advantages of table 1
6281  * and above.
6282  *
6283  * @param dev
6284  *   Pointer to Ethernet device.
6285  *
6286  * @return
6287  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6288  */
6289 struct rte_flow *
6290 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6291 {
6292         const struct rte_flow_attr attr = {
6293                 .group = 0,
6294                 .priority = 0,
6295                 .ingress = 1,
6296                 .egress = 0,
6297                 .transfer = 1,
6298         };
6299         const struct rte_flow_item pattern = {
6300                 .type = RTE_FLOW_ITEM_TYPE_END,
6301         };
6302         struct rte_flow_action_jump jump = {
6303                 .group = 1,
6304         };
6305         const struct rte_flow_action actions[] = {
6306                 {
6307                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6308                         .conf = &jump,
6309                 },
6310                 {
6311                         .type = RTE_FLOW_ACTION_TYPE_END,
6312                 },
6313         };
6314         struct mlx5_priv *priv = dev->data->dev_private;
6315         struct rte_flow_error error;
6316
6317         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
6318                                                    &attr, &pattern,
6319                                                    actions, false, &error);
6320 }
6321
6322 /**
6323  * Validate a flow supported by the NIC.
6324  *
6325  * @see rte_flow_validate()
6326  * @see rte_flow_ops
6327  */
6328 int
6329 mlx5_flow_validate(struct rte_eth_dev *dev,
6330                    const struct rte_flow_attr *attr,
6331                    const struct rte_flow_item items[],
6332                    const struct rte_flow_action original_actions[],
6333                    struct rte_flow_error *error)
6334 {
6335         int hairpin_flow;
6336         struct mlx5_translated_action_handle
6337                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6338         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6339         const struct rte_flow_action *actions;
6340         struct rte_flow_action *translated_actions = NULL;
6341         int ret = flow_action_handles_translate(dev, original_actions,
6342                                                 indir_actions,
6343                                                 &indir_actions_n,
6344                                                 &translated_actions, error);
6345
6346         if (ret)
6347                 return ret;
6348         actions = translated_actions ? translated_actions : original_actions;
6349         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6350         ret = flow_drv_validate(dev, attr, items, actions,
6351                                 true, hairpin_flow, error);
6352         rte_free(translated_actions);
6353         return ret;
6354 }
6355
6356 /**
6357  * Create a flow.
6358  *
6359  * @see rte_flow_create()
6360  * @see rte_flow_ops
6361  */
6362 struct rte_flow *
6363 mlx5_flow_create(struct rte_eth_dev *dev,
6364                  const struct rte_flow_attr *attr,
6365                  const struct rte_flow_item items[],
6366                  const struct rte_flow_action actions[],
6367                  struct rte_flow_error *error)
6368 {
6369         struct mlx5_priv *priv = dev->data->dev_private;
6370
6371         /*
6372          * If the device is not started yet, it is not allowed to created a
6373          * flow from application. PMD default flows and traffic control flows
6374          * are not affected.
6375          */
6376         if (unlikely(!dev->data->dev_started)) {
6377                 DRV_LOG(DEBUG, "port %u is not started when "
6378                         "inserting a flow", dev->data->port_id);
6379                 rte_flow_error_set(error, ENODEV,
6380                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6381                                    NULL,
6382                                    "port not started");
6383                 return NULL;
6384         }
6385
6386         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
6387                                   attr, items, actions, true, error);
6388 }
6389
6390 /**
6391  * Destroy a flow in a list.
6392  *
6393  * @param dev
6394  *   Pointer to Ethernet device.
6395  * @param list
6396  *   Pointer to the Indexed flow list. If this parameter NULL,
6397  *   there is no flow removal from the list. Be noted that as
6398  *   flow is add to the indexed list, memory of the indexed
6399  *   list points to maybe changed as flow destroyed.
6400  * @param[in] flow_idx
6401  *   Index of flow to destroy.
6402  */
6403 static void
6404 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
6405                   uint32_t flow_idx)
6406 {
6407         struct mlx5_priv *priv = dev->data->dev_private;
6408         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6409                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
6410
6411         if (!flow)
6412                 return;
6413         /*
6414          * Update RX queue flags only if port is started, otherwise it is
6415          * already clean.
6416          */
6417         if (dev->data->dev_started)
6418                 flow_rxq_flags_trim(dev, flow);
6419         flow_drv_destroy(dev, flow);
6420         if (list) {
6421                 rte_spinlock_lock(&priv->flow_list_lock);
6422                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
6423                              flow_idx, flow, next);
6424                 rte_spinlock_unlock(&priv->flow_list_lock);
6425         }
6426         if (flow->tunnel) {
6427                 struct mlx5_flow_tunnel *tunnel;
6428
6429                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6430                 RTE_VERIFY(tunnel);
6431                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6432                         mlx5_flow_tunnel_free(dev, tunnel);
6433         }
6434         flow_mreg_del_copy_action(dev, flow);
6435         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6436 }
6437
6438 /**
6439  * Destroy all flows.
6440  *
6441  * @param dev
6442  *   Pointer to Ethernet device.
6443  * @param list
6444  *   Pointer to the Indexed flow list.
6445  * @param active
6446  *   If flushing is called avtively.
6447  */
6448 void
6449 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
6450 {
6451         uint32_t num_flushed = 0;
6452
6453         while (*list) {
6454                 flow_list_destroy(dev, list, *list);
6455                 num_flushed++;
6456         }
6457         if (active) {
6458                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6459                         dev->data->port_id, num_flushed);
6460         }
6461 }
6462
6463 /**
6464  * Stop all default actions for flows.
6465  *
6466  * @param dev
6467  *   Pointer to Ethernet device.
6468  */
6469 void
6470 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6471 {
6472         flow_mreg_del_default_copy_action(dev);
6473         flow_rxq_flags_clear(dev);
6474 }
6475
6476 /**
6477  * Start all default actions for flows.
6478  *
6479  * @param dev
6480  *   Pointer to Ethernet device.
6481  * @return
6482  *   0 on success, a negative errno value otherwise and rte_errno is set.
6483  */
6484 int
6485 mlx5_flow_start_default(struct rte_eth_dev *dev)
6486 {
6487         struct rte_flow_error error;
6488
6489         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6490         return flow_mreg_add_default_copy_action(dev, &error);
6491 }
6492
6493 /**
6494  * Release key of thread specific flow workspace data.
6495  */
6496 void
6497 flow_release_workspace(void *data)
6498 {
6499         struct mlx5_flow_workspace *wks = data;
6500         struct mlx5_flow_workspace *next;
6501
6502         while (wks) {
6503                 next = wks->next;
6504                 free(wks->rss_desc.queue);
6505                 free(wks);
6506                 wks = next;
6507         }
6508 }
6509
6510 /**
6511  * Get thread specific current flow workspace.
6512  *
6513  * @return pointer to thread specific flow workspace data, NULL on error.
6514  */
6515 struct mlx5_flow_workspace*
6516 mlx5_flow_get_thread_workspace(void)
6517 {
6518         struct mlx5_flow_workspace *data;
6519
6520         data = mlx5_flow_os_get_specific_workspace();
6521         MLX5_ASSERT(data && data->inuse);
6522         if (!data || !data->inuse)
6523                 DRV_LOG(ERR, "flow workspace not initialized.");
6524         return data;
6525 }
6526
6527 /**
6528  * Allocate and init new flow workspace.
6529  *
6530  * @return pointer to flow workspace data, NULL on error.
6531  */
6532 static struct mlx5_flow_workspace*
6533 flow_alloc_thread_workspace(void)
6534 {
6535         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6536
6537         if (!data) {
6538                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6539                         "memory.");
6540                 return NULL;
6541         }
6542         data->rss_desc.queue = calloc(1,
6543                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6544         if (!data->rss_desc.queue)
6545                 goto err;
6546         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6547         return data;
6548 err:
6549         if (data->rss_desc.queue)
6550                 free(data->rss_desc.queue);
6551         free(data);
6552         return NULL;
6553 }
6554
6555 /**
6556  * Get new thread specific flow workspace.
6557  *
6558  * If current workspace inuse, create new one and set as current.
6559  *
6560  * @return pointer to thread specific flow workspace data, NULL on error.
6561  */
6562 static struct mlx5_flow_workspace*
6563 mlx5_flow_push_thread_workspace(void)
6564 {
6565         struct mlx5_flow_workspace *curr;
6566         struct mlx5_flow_workspace *data;
6567
6568         curr = mlx5_flow_os_get_specific_workspace();
6569         if (!curr) {
6570                 data = flow_alloc_thread_workspace();
6571                 if (!data)
6572                         return NULL;
6573         } else if (!curr->inuse) {
6574                 data = curr;
6575         } else if (curr->next) {
6576                 data = curr->next;
6577         } else {
6578                 data = flow_alloc_thread_workspace();
6579                 if (!data)
6580                         return NULL;
6581                 curr->next = data;
6582                 data->prev = curr;
6583         }
6584         data->inuse = 1;
6585         data->flow_idx = 0;
6586         /* Set as current workspace */
6587         if (mlx5_flow_os_set_specific_workspace(data))
6588                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6589         return data;
6590 }
6591
6592 /**
6593  * Close current thread specific flow workspace.
6594  *
6595  * If previous workspace available, set it as current.
6596  *
6597  * @return pointer to thread specific flow workspace data, NULL on error.
6598  */
6599 static void
6600 mlx5_flow_pop_thread_workspace(void)
6601 {
6602         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6603
6604         if (!data)
6605                 return;
6606         if (!data->inuse) {
6607                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6608                 return;
6609         }
6610         data->inuse = 0;
6611         if (!data->prev)
6612                 return;
6613         if (mlx5_flow_os_set_specific_workspace(data->prev))
6614                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6615 }
6616
6617 /**
6618  * Verify the flow list is empty
6619  *
6620  * @param dev
6621  *  Pointer to Ethernet device.
6622  *
6623  * @return the number of flows not released.
6624  */
6625 int
6626 mlx5_flow_verify(struct rte_eth_dev *dev)
6627 {
6628         struct mlx5_priv *priv = dev->data->dev_private;
6629         struct rte_flow *flow;
6630         uint32_t idx;
6631         int ret = 0;
6632
6633         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6634                       flow, next) {
6635                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6636                         dev->data->port_id, (void *)flow);
6637                 ++ret;
6638         }
6639         return ret;
6640 }
6641
6642 /**
6643  * Enable default hairpin egress flow.
6644  *
6645  * @param dev
6646  *   Pointer to Ethernet device.
6647  * @param queue
6648  *   The queue index.
6649  *
6650  * @return
6651  *   0 on success, a negative errno value otherwise and rte_errno is set.
6652  */
6653 int
6654 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6655                             uint32_t queue)
6656 {
6657         struct mlx5_priv *priv = dev->data->dev_private;
6658         const struct rte_flow_attr attr = {
6659                 .egress = 1,
6660                 .priority = 0,
6661         };
6662         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6663                 .queue = queue,
6664         };
6665         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6666                 .queue = UINT32_MAX,
6667         };
6668         struct rte_flow_item items[] = {
6669                 {
6670                         .type = (enum rte_flow_item_type)
6671                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6672                         .spec = &queue_spec,
6673                         .last = NULL,
6674                         .mask = &queue_mask,
6675                 },
6676                 {
6677                         .type = RTE_FLOW_ITEM_TYPE_END,
6678                 },
6679         };
6680         struct rte_flow_action_jump jump = {
6681                 .group = MLX5_HAIRPIN_TX_TABLE,
6682         };
6683         struct rte_flow_action actions[2];
6684         uint32_t flow_idx;
6685         struct rte_flow_error error;
6686
6687         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6688         actions[0].conf = &jump;
6689         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6690         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6691                                 &attr, items, actions, false, &error);
6692         if (!flow_idx) {
6693                 DRV_LOG(DEBUG,
6694                         "Failed to create ctrl flow: rte_errno(%d),"
6695                         " type(%d), message(%s)",
6696                         rte_errno, error.type,
6697                         error.message ? error.message : " (no stated reason)");
6698                 return -rte_errno;
6699         }
6700         return 0;
6701 }
6702
6703 /**
6704  * Enable a control flow configured from the control plane.
6705  *
6706  * @param dev
6707  *   Pointer to Ethernet device.
6708  * @param eth_spec
6709  *   An Ethernet flow spec to apply.
6710  * @param eth_mask
6711  *   An Ethernet flow mask to apply.
6712  * @param vlan_spec
6713  *   A VLAN flow spec to apply.
6714  * @param vlan_mask
6715  *   A VLAN flow mask to apply.
6716  *
6717  * @return
6718  *   0 on success, a negative errno value otherwise and rte_errno is set.
6719  */
6720 int
6721 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6722                     struct rte_flow_item_eth *eth_spec,
6723                     struct rte_flow_item_eth *eth_mask,
6724                     struct rte_flow_item_vlan *vlan_spec,
6725                     struct rte_flow_item_vlan *vlan_mask)
6726 {
6727         struct mlx5_priv *priv = dev->data->dev_private;
6728         const struct rte_flow_attr attr = {
6729                 .ingress = 1,
6730                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6731         };
6732         struct rte_flow_item items[] = {
6733                 {
6734                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6735                         .spec = eth_spec,
6736                         .last = NULL,
6737                         .mask = eth_mask,
6738                 },
6739                 {
6740                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6741                                               RTE_FLOW_ITEM_TYPE_END,
6742                         .spec = vlan_spec,
6743                         .last = NULL,
6744                         .mask = vlan_mask,
6745                 },
6746                 {
6747                         .type = RTE_FLOW_ITEM_TYPE_END,
6748                 },
6749         };
6750         uint16_t queue[priv->reta_idx_n];
6751         struct rte_flow_action_rss action_rss = {
6752                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6753                 .level = 0,
6754                 .types = priv->rss_conf.rss_hf,
6755                 .key_len = priv->rss_conf.rss_key_len,
6756                 .queue_num = priv->reta_idx_n,
6757                 .key = priv->rss_conf.rss_key,
6758                 .queue = queue,
6759         };
6760         struct rte_flow_action actions[] = {
6761                 {
6762                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6763                         .conf = &action_rss,
6764                 },
6765                 {
6766                         .type = RTE_FLOW_ACTION_TYPE_END,
6767                 },
6768         };
6769         uint32_t flow_idx;
6770         struct rte_flow_error error;
6771         unsigned int i;
6772
6773         if (!priv->reta_idx_n || !priv->rxqs_n) {
6774                 return 0;
6775         }
6776         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6777                 action_rss.types = 0;
6778         for (i = 0; i != priv->reta_idx_n; ++i)
6779                 queue[i] = (*priv->reta_idx)[i];
6780         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6781                                 &attr, items, actions, false, &error);
6782         if (!flow_idx)
6783                 return -rte_errno;
6784         return 0;
6785 }
6786
6787 /**
6788  * Enable a flow control configured from the control plane.
6789  *
6790  * @param dev
6791  *   Pointer to Ethernet device.
6792  * @param eth_spec
6793  *   An Ethernet flow spec to apply.
6794  * @param eth_mask
6795  *   An Ethernet flow mask to apply.
6796  *
6797  * @return
6798  *   0 on success, a negative errno value otherwise and rte_errno is set.
6799  */
6800 int
6801 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6802                struct rte_flow_item_eth *eth_spec,
6803                struct rte_flow_item_eth *eth_mask)
6804 {
6805         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6806 }
6807
6808 /**
6809  * Create default miss flow rule matching lacp traffic
6810  *
6811  * @param dev
6812  *   Pointer to Ethernet device.
6813  * @param eth_spec
6814  *   An Ethernet flow spec to apply.
6815  *
6816  * @return
6817  *   0 on success, a negative errno value otherwise and rte_errno is set.
6818  */
6819 int
6820 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6821 {
6822         struct mlx5_priv *priv = dev->data->dev_private;
6823         /*
6824          * The LACP matching is done by only using ether type since using
6825          * a multicast dst mac causes kernel to give low priority to this flow.
6826          */
6827         static const struct rte_flow_item_eth lacp_spec = {
6828                 .type = RTE_BE16(0x8809),
6829         };
6830         static const struct rte_flow_item_eth lacp_mask = {
6831                 .type = 0xffff,
6832         };
6833         const struct rte_flow_attr attr = {
6834                 .ingress = 1,
6835         };
6836         struct rte_flow_item items[] = {
6837                 {
6838                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6839                         .spec = &lacp_spec,
6840                         .mask = &lacp_mask,
6841                 },
6842                 {
6843                         .type = RTE_FLOW_ITEM_TYPE_END,
6844                 },
6845         };
6846         struct rte_flow_action actions[] = {
6847                 {
6848                         .type = (enum rte_flow_action_type)
6849                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6850                 },
6851                 {
6852                         .type = RTE_FLOW_ACTION_TYPE_END,
6853                 },
6854         };
6855         struct rte_flow_error error;
6856         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6857                                 &attr, items, actions, false, &error);
6858
6859         if (!flow_idx)
6860                 return -rte_errno;
6861         return 0;
6862 }
6863
6864 /**
6865  * Destroy a flow.
6866  *
6867  * @see rte_flow_destroy()
6868  * @see rte_flow_ops
6869  */
6870 int
6871 mlx5_flow_destroy(struct rte_eth_dev *dev,
6872                   struct rte_flow *flow,
6873                   struct rte_flow_error *error __rte_unused)
6874 {
6875         struct mlx5_priv *priv = dev->data->dev_private;
6876
6877         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6878         return 0;
6879 }
6880
6881 /**
6882  * Destroy all flows.
6883  *
6884  * @see rte_flow_flush()
6885  * @see rte_flow_ops
6886  */
6887 int
6888 mlx5_flow_flush(struct rte_eth_dev *dev,
6889                 struct rte_flow_error *error __rte_unused)
6890 {
6891         struct mlx5_priv *priv = dev->data->dev_private;
6892
6893         mlx5_flow_list_flush(dev, &priv->flows, false);
6894         return 0;
6895 }
6896
6897 /**
6898  * Isolated mode.
6899  *
6900  * @see rte_flow_isolate()
6901  * @see rte_flow_ops
6902  */
6903 int
6904 mlx5_flow_isolate(struct rte_eth_dev *dev,
6905                   int enable,
6906                   struct rte_flow_error *error)
6907 {
6908         struct mlx5_priv *priv = dev->data->dev_private;
6909
6910         if (dev->data->dev_started) {
6911                 rte_flow_error_set(error, EBUSY,
6912                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6913                                    NULL,
6914                                    "port must be stopped first");
6915                 return -rte_errno;
6916         }
6917         priv->isolated = !!enable;
6918         if (enable)
6919                 dev->dev_ops = &mlx5_dev_ops_isolate;
6920         else
6921                 dev->dev_ops = &mlx5_dev_ops;
6922
6923         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6924         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6925
6926         return 0;
6927 }
6928
6929 /**
6930  * Query a flow.
6931  *
6932  * @see rte_flow_query()
6933  * @see rte_flow_ops
6934  */
6935 static int
6936 flow_drv_query(struct rte_eth_dev *dev,
6937                uint32_t flow_idx,
6938                const struct rte_flow_action *actions,
6939                void *data,
6940                struct rte_flow_error *error)
6941 {
6942         struct mlx5_priv *priv = dev->data->dev_private;
6943         const struct mlx5_flow_driver_ops *fops;
6944         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6945                                                [MLX5_IPOOL_RTE_FLOW],
6946                                                flow_idx);
6947         enum mlx5_flow_drv_type ftype;
6948
6949         if (!flow) {
6950                 return rte_flow_error_set(error, ENOENT,
6951                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6952                           NULL,
6953                           "invalid flow handle");
6954         }
6955         ftype = flow->drv_type;
6956         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6957         fops = flow_get_drv_ops(ftype);
6958
6959         return fops->query(dev, flow, actions, data, error);
6960 }
6961
6962 /**
6963  * Query a flow.
6964  *
6965  * @see rte_flow_query()
6966  * @see rte_flow_ops
6967  */
6968 int
6969 mlx5_flow_query(struct rte_eth_dev *dev,
6970                 struct rte_flow *flow,
6971                 const struct rte_flow_action *actions,
6972                 void *data,
6973                 struct rte_flow_error *error)
6974 {
6975         int ret;
6976
6977         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6978                              error);
6979         if (ret < 0)
6980                 return ret;
6981         return 0;
6982 }
6983
6984 /**
6985  * Get rte_flow callbacks.
6986  *
6987  * @param dev
6988  *   Pointer to Ethernet device structure.
6989  * @param ops
6990  *   Pointer to operation-specific structure.
6991  *
6992  * @return 0
6993  */
6994 int
6995 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
6996                   const struct rte_flow_ops **ops)
6997 {
6998         *ops = &mlx5_flow_ops;
6999         return 0;
7000 }
7001
7002 /**
7003  * Validate meter policy actions.
7004  * Dispatcher for action type specific validation.
7005  *
7006  * @param[in] dev
7007  *   Pointer to the Ethernet device structure.
7008  * @param[in] action
7009  *   The meter policy action object to validate.
7010  * @param[in] attr
7011  *   Attributes of flow to determine steering domain.
7012  * @param[out] is_rss
7013  *   Is RSS or not.
7014  * @param[out] domain_bitmap
7015  *   Domain bitmap.
7016  * @param[out] is_def_policy
7017  *   Is default policy or not.
7018  * @param[out] error
7019  *   Perform verbose error reporting if not NULL. Initialized in case of
7020  *   error only.
7021  *
7022  * @return
7023  *   0 on success, otherwise negative errno value.
7024  */
7025 int
7026 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7027                         const struct rte_flow_action *actions[RTE_COLORS],
7028                         struct rte_flow_attr *attr,
7029                         bool *is_rss,
7030                         uint8_t *domain_bitmap,
7031                         bool *is_def_policy,
7032                         struct rte_mtr_error *error)
7033 {
7034         const struct mlx5_flow_driver_ops *fops;
7035
7036         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7037         return fops->validate_mtr_acts(dev, actions, attr,
7038                         is_rss, domain_bitmap, is_def_policy, error);
7039 }
7040
7041 /**
7042  * Destroy the meter table set.
7043  *
7044  * @param[in] dev
7045  *   Pointer to Ethernet device.
7046  * @param[in] mtr_policy
7047  *   Meter policy struct.
7048  */
7049 void
7050 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7051                       struct mlx5_flow_meter_policy *mtr_policy)
7052 {
7053         const struct mlx5_flow_driver_ops *fops;
7054
7055         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7056         fops->destroy_mtr_acts(dev, mtr_policy);
7057 }
7058
7059 /**
7060  * Create policy action, lock free,
7061  * (mutex should be acquired by caller).
7062  * Dispatcher for action type specific call.
7063  *
7064  * @param[in] dev
7065  *   Pointer to the Ethernet device structure.
7066  * @param[in] mtr_policy
7067  *   Meter policy struct.
7068  * @param[in] action
7069  *   Action specification used to create meter actions.
7070  * @param[out] error
7071  *   Perform verbose error reporting if not NULL. Initialized in case of
7072  *   error only.
7073  *
7074  * @return
7075  *   0 on success, otherwise negative errno value.
7076  */
7077 int
7078 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7079                       struct mlx5_flow_meter_policy *mtr_policy,
7080                       const struct rte_flow_action *actions[RTE_COLORS],
7081                       struct rte_mtr_error *error)
7082 {
7083         const struct mlx5_flow_driver_ops *fops;
7084
7085         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7086         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7087 }
7088
7089 /**
7090  * Create policy rules, lock free,
7091  * (mutex should be acquired by caller).
7092  * Dispatcher for action type specific call.
7093  *
7094  * @param[in] dev
7095  *   Pointer to the Ethernet device structure.
7096  * @param[in] mtr_policy
7097  *   Meter policy struct.
7098  *
7099  * @return
7100  *   0 on success, -1 otherwise.
7101  */
7102 int
7103 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7104                              struct mlx5_flow_meter_policy *mtr_policy)
7105 {
7106         const struct mlx5_flow_driver_ops *fops;
7107
7108         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7109         return fops->create_policy_rules(dev, mtr_policy);
7110 }
7111
7112 /**
7113  * Destroy policy rules, lock free,
7114  * (mutex should be acquired by caller).
7115  * Dispatcher for action type specific call.
7116  *
7117  * @param[in] dev
7118  *   Pointer to the Ethernet device structure.
7119  * @param[in] mtr_policy
7120  *   Meter policy struct.
7121  */
7122 void
7123 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7124                              struct mlx5_flow_meter_policy *mtr_policy)
7125 {
7126         const struct mlx5_flow_driver_ops *fops;
7127
7128         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7129         fops->destroy_policy_rules(dev, mtr_policy);
7130 }
7131
7132 /**
7133  * Destroy the default policy table set.
7134  *
7135  * @param[in] dev
7136  *   Pointer to Ethernet device.
7137  */
7138 void
7139 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7140 {
7141         const struct mlx5_flow_driver_ops *fops;
7142
7143         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7144         fops->destroy_def_policy(dev);
7145 }
7146
7147 /**
7148  * Destroy the default policy table set.
7149  *
7150  * @param[in] dev
7151  *   Pointer to Ethernet device.
7152  *
7153  * @return
7154  *   0 on success, -1 otherwise.
7155  */
7156 int
7157 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7158 {
7159         const struct mlx5_flow_driver_ops *fops;
7160
7161         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7162         return fops->create_def_policy(dev);
7163 }
7164
7165 /**
7166  * Create the needed meter and suffix tables.
7167  *
7168  * @param[in] dev
7169  *   Pointer to Ethernet device.
7170  *
7171  * @return
7172  *   0 on success, -1 otherwise.
7173  */
7174 int
7175 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7176                         struct mlx5_flow_meter_info *fm,
7177                         uint32_t mtr_idx,
7178                         uint8_t domain_bitmap)
7179 {
7180         const struct mlx5_flow_driver_ops *fops;
7181
7182         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7183         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7184 }
7185
7186 /**
7187  * Destroy the meter table set.
7188  *
7189  * @param[in] dev
7190  *   Pointer to Ethernet device.
7191  * @param[in] tbl
7192  *   Pointer to the meter table set.
7193  */
7194 void
7195 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7196                            struct mlx5_flow_meter_info *fm)
7197 {
7198         const struct mlx5_flow_driver_ops *fops;
7199
7200         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7201         fops->destroy_mtr_tbls(dev, fm);
7202 }
7203
7204 /**
7205  * Destroy the global meter drop table.
7206  *
7207  * @param[in] dev
7208  *   Pointer to Ethernet device.
7209  */
7210 void
7211 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7212 {
7213         const struct mlx5_flow_driver_ops *fops;
7214
7215         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7216         fops->destroy_mtr_drop_tbls(dev);
7217 }
7218
7219 /**
7220  * Allocate the needed aso flow meter id.
7221  *
7222  * @param[in] dev
7223  *   Pointer to Ethernet device.
7224  *
7225  * @return
7226  *   Index to aso flow meter on success, NULL otherwise.
7227  */
7228 uint32_t
7229 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7230 {
7231         const struct mlx5_flow_driver_ops *fops;
7232
7233         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7234         return fops->create_meter(dev);
7235 }
7236
7237 /**
7238  * Free the aso flow meter id.
7239  *
7240  * @param[in] dev
7241  *   Pointer to Ethernet device.
7242  * @param[in] mtr_idx
7243  *  Index to aso flow meter to be free.
7244  *
7245  * @return
7246  *   0 on success.
7247  */
7248 void
7249 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7250 {
7251         const struct mlx5_flow_driver_ops *fops;
7252
7253         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7254         fops->free_meter(dev, mtr_idx);
7255 }
7256
7257 /**
7258  * Allocate a counter.
7259  *
7260  * @param[in] dev
7261  *   Pointer to Ethernet device structure.
7262  *
7263  * @return
7264  *   Index to allocated counter  on success, 0 otherwise.
7265  */
7266 uint32_t
7267 mlx5_counter_alloc(struct rte_eth_dev *dev)
7268 {
7269         const struct mlx5_flow_driver_ops *fops;
7270         struct rte_flow_attr attr = { .transfer = 0 };
7271
7272         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7273                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7274                 return fops->counter_alloc(dev);
7275         }
7276         DRV_LOG(ERR,
7277                 "port %u counter allocate is not supported.",
7278                  dev->data->port_id);
7279         return 0;
7280 }
7281
7282 /**
7283  * Free a counter.
7284  *
7285  * @param[in] dev
7286  *   Pointer to Ethernet device structure.
7287  * @param[in] cnt
7288  *   Index to counter to be free.
7289  */
7290 void
7291 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7292 {
7293         const struct mlx5_flow_driver_ops *fops;
7294         struct rte_flow_attr attr = { .transfer = 0 };
7295
7296         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7297                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7298                 fops->counter_free(dev, cnt);
7299                 return;
7300         }
7301         DRV_LOG(ERR,
7302                 "port %u counter free is not supported.",
7303                  dev->data->port_id);
7304 }
7305
7306 /**
7307  * Query counter statistics.
7308  *
7309  * @param[in] dev
7310  *   Pointer to Ethernet device structure.
7311  * @param[in] cnt
7312  *   Index to counter to query.
7313  * @param[in] clear
7314  *   Set to clear counter statistics.
7315  * @param[out] pkts
7316  *   The counter hits packets number to save.
7317  * @param[out] bytes
7318  *   The counter hits bytes number to save.
7319  *
7320  * @return
7321  *   0 on success, a negative errno value otherwise.
7322  */
7323 int
7324 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7325                    bool clear, uint64_t *pkts, uint64_t *bytes)
7326 {
7327         const struct mlx5_flow_driver_ops *fops;
7328         struct rte_flow_attr attr = { .transfer = 0 };
7329
7330         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7331                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7332                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7333         }
7334         DRV_LOG(ERR,
7335                 "port %u counter query is not supported.",
7336                  dev->data->port_id);
7337         return -ENOTSUP;
7338 }
7339
7340 /**
7341  * Allocate a new memory for the counter values wrapped by all the needed
7342  * management.
7343  *
7344  * @param[in] sh
7345  *   Pointer to mlx5_dev_ctx_shared object.
7346  *
7347  * @return
7348  *   0 on success, a negative errno value otherwise.
7349  */
7350 static int
7351 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7352 {
7353         struct mlx5_devx_mkey_attr mkey_attr;
7354         struct mlx5_counter_stats_mem_mng *mem_mng;
7355         volatile struct flow_counter_stats *raw_data;
7356         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7357         int size = (sizeof(struct flow_counter_stats) *
7358                         MLX5_COUNTERS_PER_POOL +
7359                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7360                         sizeof(struct mlx5_counter_stats_mem_mng);
7361         size_t pgsize = rte_mem_page_size();
7362         uint8_t *mem;
7363         int i;
7364
7365         if (pgsize == (size_t)-1) {
7366                 DRV_LOG(ERR, "Failed to get mem page size");
7367                 rte_errno = ENOMEM;
7368                 return -ENOMEM;
7369         }
7370         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7371         if (!mem) {
7372                 rte_errno = ENOMEM;
7373                 return -ENOMEM;
7374         }
7375         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7376         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7377         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7378                                                  IBV_ACCESS_LOCAL_WRITE);
7379         if (!mem_mng->umem) {
7380                 rte_errno = errno;
7381                 mlx5_free(mem);
7382                 return -rte_errno;
7383         }
7384         memset(&mkey_attr, 0, sizeof(mkey_attr));
7385         mkey_attr.addr = (uintptr_t)mem;
7386         mkey_attr.size = size;
7387         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7388         mkey_attr.pd = sh->pdn;
7389         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7390         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7391         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7392         if (!mem_mng->dm) {
7393                 mlx5_os_umem_dereg(mem_mng->umem);
7394                 rte_errno = errno;
7395                 mlx5_free(mem);
7396                 return -rte_errno;
7397         }
7398         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7399         raw_data = (volatile struct flow_counter_stats *)mem;
7400         for (i = 0; i < raws_n; ++i) {
7401                 mem_mng->raws[i].mem_mng = mem_mng;
7402                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7403         }
7404         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7405                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7406                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7407                                  next);
7408         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7409         sh->cmng.mem_mng = mem_mng;
7410         return 0;
7411 }
7412
7413 /**
7414  * Set the statistic memory to the new counter pool.
7415  *
7416  * @param[in] sh
7417  *   Pointer to mlx5_dev_ctx_shared object.
7418  * @param[in] pool
7419  *   Pointer to the pool to set the statistic memory.
7420  *
7421  * @return
7422  *   0 on success, a negative errno value otherwise.
7423  */
7424 static int
7425 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7426                                struct mlx5_flow_counter_pool *pool)
7427 {
7428         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7429         /* Resize statistic memory once used out. */
7430         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7431             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7432                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7433                 return -1;
7434         }
7435         rte_spinlock_lock(&pool->sl);
7436         pool->raw = cmng->mem_mng->raws + pool->index %
7437                     MLX5_CNT_CONTAINER_RESIZE;
7438         rte_spinlock_unlock(&pool->sl);
7439         pool->raw_hw = NULL;
7440         return 0;
7441 }
7442
7443 #define MLX5_POOL_QUERY_FREQ_US 1000000
7444
7445 /**
7446  * Set the periodic procedure for triggering asynchronous batch queries for all
7447  * the counter pools.
7448  *
7449  * @param[in] sh
7450  *   Pointer to mlx5_dev_ctx_shared object.
7451  */
7452 void
7453 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7454 {
7455         uint32_t pools_n, us;
7456
7457         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7458         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7459         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7460         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7461                 sh->cmng.query_thread_on = 0;
7462                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7463         } else {
7464                 sh->cmng.query_thread_on = 1;
7465         }
7466 }
7467
7468 /**
7469  * The periodic procedure for triggering asynchronous batch queries for all the
7470  * counter pools. This function is probably called by the host thread.
7471  *
7472  * @param[in] arg
7473  *   The parameter for the alarm process.
7474  */
7475 void
7476 mlx5_flow_query_alarm(void *arg)
7477 {
7478         struct mlx5_dev_ctx_shared *sh = arg;
7479         int ret;
7480         uint16_t pool_index = sh->cmng.pool_index;
7481         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7482         struct mlx5_flow_counter_pool *pool;
7483         uint16_t n_valid;
7484
7485         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7486                 goto set_alarm;
7487         rte_spinlock_lock(&cmng->pool_update_sl);
7488         pool = cmng->pools[pool_index];
7489         n_valid = cmng->n_valid;
7490         rte_spinlock_unlock(&cmng->pool_update_sl);
7491         /* Set the statistic memory to the new created pool. */
7492         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7493                 goto set_alarm;
7494         if (pool->raw_hw)
7495                 /* There is a pool query in progress. */
7496                 goto set_alarm;
7497         pool->raw_hw =
7498                 LIST_FIRST(&sh->cmng.free_stat_raws);
7499         if (!pool->raw_hw)
7500                 /* No free counter statistics raw memory. */
7501                 goto set_alarm;
7502         /*
7503          * Identify the counters released between query trigger and query
7504          * handle more efficiently. The counter released in this gap period
7505          * should wait for a new round of query as the new arrived packets
7506          * will not be taken into account.
7507          */
7508         pool->query_gen++;
7509         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7510                                                MLX5_COUNTERS_PER_POOL,
7511                                                NULL, NULL,
7512                                                pool->raw_hw->mem_mng->dm->id,
7513                                                (void *)(uintptr_t)
7514                                                pool->raw_hw->data,
7515                                                sh->devx_comp,
7516                                                (uint64_t)(uintptr_t)pool);
7517         if (ret) {
7518                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7519                         " %d", pool->min_dcs->id);
7520                 pool->raw_hw = NULL;
7521                 goto set_alarm;
7522         }
7523         LIST_REMOVE(pool->raw_hw, next);
7524         sh->cmng.pending_queries++;
7525         pool_index++;
7526         if (pool_index >= n_valid)
7527                 pool_index = 0;
7528 set_alarm:
7529         sh->cmng.pool_index = pool_index;
7530         mlx5_set_query_alarm(sh);
7531 }
7532
7533 /**
7534  * Check and callback event for new aged flow in the counter pool
7535  *
7536  * @param[in] sh
7537  *   Pointer to mlx5_dev_ctx_shared object.
7538  * @param[in] pool
7539  *   Pointer to Current counter pool.
7540  */
7541 static void
7542 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7543                    struct mlx5_flow_counter_pool *pool)
7544 {
7545         struct mlx5_priv *priv;
7546         struct mlx5_flow_counter *cnt;
7547         struct mlx5_age_info *age_info;
7548         struct mlx5_age_param *age_param;
7549         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7550         struct mlx5_counter_stats_raw *prev = pool->raw;
7551         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7552         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7553         uint16_t expected = AGE_CANDIDATE;
7554         uint32_t i;
7555
7556         pool->time_of_last_age_check = curr_time;
7557         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7558                 cnt = MLX5_POOL_GET_CNT(pool, i);
7559                 age_param = MLX5_CNT_TO_AGE(cnt);
7560                 if (__atomic_load_n(&age_param->state,
7561                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7562                         continue;
7563                 if (cur->data[i].hits != prev->data[i].hits) {
7564                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7565                                          __ATOMIC_RELAXED);
7566                         continue;
7567                 }
7568                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7569                                        time_delta,
7570                                        __ATOMIC_RELAXED) <= age_param->timeout)
7571                         continue;
7572                 /**
7573                  * Hold the lock first, or if between the
7574                  * state AGE_TMOUT and tailq operation the
7575                  * release happened, the release procedure
7576                  * may delete a non-existent tailq node.
7577                  */
7578                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7579                 age_info = GET_PORT_AGE_INFO(priv);
7580                 rte_spinlock_lock(&age_info->aged_sl);
7581                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7582                                                 AGE_TMOUT, false,
7583                                                 __ATOMIC_RELAXED,
7584                                                 __ATOMIC_RELAXED)) {
7585                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7586                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7587                 }
7588                 rte_spinlock_unlock(&age_info->aged_sl);
7589         }
7590         mlx5_age_event_prepare(sh);
7591 }
7592
7593 /**
7594  * Handler for the HW respond about ready values from an asynchronous batch
7595  * query. This function is probably called by the host thread.
7596  *
7597  * @param[in] sh
7598  *   The pointer to the shared device context.
7599  * @param[in] async_id
7600  *   The Devx async ID.
7601  * @param[in] status
7602  *   The status of the completion.
7603  */
7604 void
7605 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7606                                   uint64_t async_id, int status)
7607 {
7608         struct mlx5_flow_counter_pool *pool =
7609                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7610         struct mlx5_counter_stats_raw *raw_to_free;
7611         uint8_t query_gen = pool->query_gen ^ 1;
7612         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7613         enum mlx5_counter_type cnt_type =
7614                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7615                                 MLX5_COUNTER_TYPE_ORIGIN;
7616
7617         if (unlikely(status)) {
7618                 raw_to_free = pool->raw_hw;
7619         } else {
7620                 raw_to_free = pool->raw;
7621                 if (pool->is_aged)
7622                         mlx5_flow_aging_check(sh, pool);
7623                 rte_spinlock_lock(&pool->sl);
7624                 pool->raw = pool->raw_hw;
7625                 rte_spinlock_unlock(&pool->sl);
7626                 /* Be sure the new raw counters data is updated in memory. */
7627                 rte_io_wmb();
7628                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7629                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7630                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7631                                      &pool->counters[query_gen], next);
7632                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7633                 }
7634         }
7635         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7636         pool->raw_hw = NULL;
7637         sh->cmng.pending_queries--;
7638 }
7639
7640 static int
7641 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7642                     const struct flow_grp_info *grp_info,
7643                     struct rte_flow_error *error)
7644 {
7645         if (grp_info->transfer && grp_info->external &&
7646             grp_info->fdb_def_rule) {
7647                 if (group == UINT32_MAX)
7648                         return rte_flow_error_set
7649                                                 (error, EINVAL,
7650                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7651                                                  NULL,
7652                                                  "group index not supported");
7653                 *table = group + 1;
7654         } else {
7655                 *table = group;
7656         }
7657         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7658         return 0;
7659 }
7660
7661 /**
7662  * Translate the rte_flow group index to HW table value.
7663  *
7664  * If tunnel offload is disabled, all group ids converted to flow table
7665  * id using the standard method.
7666  * If tunnel offload is enabled, group id can be converted using the
7667  * standard or tunnel conversion method. Group conversion method
7668  * selection depends on flags in `grp_info` parameter:
7669  * - Internal (grp_info.external == 0) groups conversion uses the
7670  *   standard method.
7671  * - Group ids in JUMP action converted with the tunnel conversion.
7672  * - Group id in rule attribute conversion depends on a rule type and
7673  *   group id value:
7674  *   ** non zero group attributes converted with the tunnel method
7675  *   ** zero group attribute in non-tunnel rule is converted using the
7676  *      standard method - there's only one root table
7677  *   ** zero group attribute in steer tunnel rule is converted with the
7678  *      standard method - single root table
7679  *   ** zero group attribute in match tunnel rule is a special OvS
7680  *      case: that value is used for portability reasons. That group
7681  *      id is converted with the tunnel conversion method.
7682  *
7683  * @param[in] dev
7684  *   Port device
7685  * @param[in] tunnel
7686  *   PMD tunnel offload object
7687  * @param[in] group
7688  *   rte_flow group index value.
7689  * @param[out] table
7690  *   HW table value.
7691  * @param[in] grp_info
7692  *   flags used for conversion
7693  * @param[out] error
7694  *   Pointer to error structure.
7695  *
7696  * @return
7697  *   0 on success, a negative errno value otherwise and rte_errno is set.
7698  */
7699 int
7700 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7701                          const struct mlx5_flow_tunnel *tunnel,
7702                          uint32_t group, uint32_t *table,
7703                          const struct flow_grp_info *grp_info,
7704                          struct rte_flow_error *error)
7705 {
7706         int ret;
7707         bool standard_translation;
7708
7709         if (!grp_info->skip_scale && grp_info->external &&
7710             group < MLX5_MAX_TABLES_EXTERNAL)
7711                 group *= MLX5_FLOW_TABLE_FACTOR;
7712         if (is_tunnel_offload_active(dev)) {
7713                 standard_translation = !grp_info->external ||
7714                                         grp_info->std_tbl_fix;
7715         } else {
7716                 standard_translation = true;
7717         }
7718         DRV_LOG(DEBUG,
7719                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7720                 dev->data->port_id, group, grp_info->transfer,
7721                 grp_info->external, grp_info->fdb_def_rule,
7722                 standard_translation ? "STANDARD" : "TUNNEL");
7723         if (standard_translation)
7724                 ret = flow_group_to_table(dev->data->port_id, group, table,
7725                                           grp_info, error);
7726         else
7727                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7728                                                       table, error);
7729
7730         return ret;
7731 }
7732
7733 /**
7734  * Discover availability of metadata reg_c's.
7735  *
7736  * Iteratively use test flows to check availability.
7737  *
7738  * @param[in] dev
7739  *   Pointer to the Ethernet device structure.
7740  *
7741  * @return
7742  *   0 on success, a negative errno value otherwise and rte_errno is set.
7743  */
7744 int
7745 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7746 {
7747         struct mlx5_priv *priv = dev->data->dev_private;
7748         struct mlx5_dev_config *config = &priv->config;
7749         enum modify_reg idx;
7750         int n = 0;
7751
7752         /* reg_c[0] and reg_c[1] are reserved. */
7753         config->flow_mreg_c[n++] = REG_C_0;
7754         config->flow_mreg_c[n++] = REG_C_1;
7755         /* Discover availability of other reg_c's. */
7756         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7757                 struct rte_flow_attr attr = {
7758                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7759                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7760                         .ingress = 1,
7761                 };
7762                 struct rte_flow_item items[] = {
7763                         [0] = {
7764                                 .type = RTE_FLOW_ITEM_TYPE_END,
7765                         },
7766                 };
7767                 struct rte_flow_action actions[] = {
7768                         [0] = {
7769                                 .type = (enum rte_flow_action_type)
7770                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7771                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7772                                         .src = REG_C_1,
7773                                         .dst = idx,
7774                                 },
7775                         },
7776                         [1] = {
7777                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7778                                 .conf = &(struct rte_flow_action_jump){
7779                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7780                                 },
7781                         },
7782                         [2] = {
7783                                 .type = RTE_FLOW_ACTION_TYPE_END,
7784                         },
7785                 };
7786                 uint32_t flow_idx;
7787                 struct rte_flow *flow;
7788                 struct rte_flow_error error;
7789
7790                 if (!config->dv_flow_en)
7791                         break;
7792                 /* Create internal flow, validation skips copy action. */
7793                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7794                                             actions, false, &error);
7795                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7796                                       flow_idx);
7797                 if (!flow)
7798                         continue;
7799                 config->flow_mreg_c[n++] = idx;
7800                 flow_list_destroy(dev, NULL, flow_idx);
7801         }
7802         for (; n < MLX5_MREG_C_NUM; ++n)
7803                 config->flow_mreg_c[n] = REG_NON;
7804         return 0;
7805 }
7806
7807 /**
7808  * Dump flow raw hw data to file
7809  *
7810  * @param[in] dev
7811  *    The pointer to Ethernet device.
7812  * @param[in] file
7813  *   A pointer to a file for output.
7814  * @param[out] error
7815  *   Perform verbose error reporting if not NULL. PMDs initialize this
7816  *   structure in case of error only.
7817  * @return
7818  *   0 on success, a nagative value otherwise.
7819  */
7820 int
7821 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
7822                    FILE *file,
7823                    struct rte_flow_error *error __rte_unused)
7824 {
7825         struct mlx5_priv *priv = dev->data->dev_private;
7826         struct mlx5_dev_ctx_shared *sh = priv->sh;
7827         uint32_t handle_idx;
7828         int ret;
7829         struct mlx5_flow_handle *dh;
7830         struct rte_flow *flow;
7831
7832         if (!priv->config.dv_flow_en) {
7833                 if (fputs("device dv flow disabled\n", file) <= 0)
7834                         return -errno;
7835                 return -ENOTSUP;
7836         }
7837
7838         /* dump all */
7839         if (!flow_idx)
7840                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
7841                                         sh->rx_domain,
7842                                         sh->tx_domain, file);
7843         /* dump one */
7844         flow = mlx5_ipool_get(priv->sh->ipool
7845                         [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
7846         if (!flow)
7847                 return -ENOENT;
7848
7849         handle_idx = flow->dev_handles;
7850         while (handle_idx) {
7851                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7852                                 handle_idx);
7853                 if (!dh)
7854                         return -ENOENT;
7855                 if (dh->drv_flow) {
7856                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
7857                                         file);
7858                         if (ret)
7859                                 return -ENOENT;
7860                 }
7861                 handle_idx = dh->next.next;
7862         }
7863         return 0;
7864 }
7865
7866 /**
7867  * Get aged-out flows.
7868  *
7869  * @param[in] dev
7870  *   Pointer to the Ethernet device structure.
7871  * @param[in] context
7872  *   The address of an array of pointers to the aged-out flows contexts.
7873  * @param[in] nb_countexts
7874  *   The length of context array pointers.
7875  * @param[out] error
7876  *   Perform verbose error reporting if not NULL. Initialized in case of
7877  *   error only.
7878  *
7879  * @return
7880  *   how many contexts get in success, otherwise negative errno value.
7881  *   if nb_contexts is 0, return the amount of all aged contexts.
7882  *   if nb_contexts is not 0 , return the amount of aged flows reported
7883  *   in the context array.
7884  */
7885 int
7886 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
7887                         uint32_t nb_contexts, struct rte_flow_error *error)
7888 {
7889         const struct mlx5_flow_driver_ops *fops;
7890         struct rte_flow_attr attr = { .transfer = 0 };
7891
7892         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7893                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7894                 return fops->get_aged_flows(dev, contexts, nb_contexts,
7895                                                     error);
7896         }
7897         DRV_LOG(ERR,
7898                 "port %u get aged flows is not supported.",
7899                  dev->data->port_id);
7900         return -ENOTSUP;
7901 }
7902
7903 /* Wrapper for driver action_validate op callback */
7904 static int
7905 flow_drv_action_validate(struct rte_eth_dev *dev,
7906                          const struct rte_flow_indir_action_conf *conf,
7907                          const struct rte_flow_action *action,
7908                          const struct mlx5_flow_driver_ops *fops,
7909                          struct rte_flow_error *error)
7910 {
7911         static const char err_msg[] = "indirect action validation unsupported";
7912
7913         if (!fops->action_validate) {
7914                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7915                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7916                                    NULL, err_msg);
7917                 return -rte_errno;
7918         }
7919         return fops->action_validate(dev, conf, action, error);
7920 }
7921
7922 /**
7923  * Destroys the shared action by handle.
7924  *
7925  * @param dev
7926  *   Pointer to Ethernet device structure.
7927  * @param[in] handle
7928  *   Handle for the indirect action object to be destroyed.
7929  * @param[out] error
7930  *   Perform verbose error reporting if not NULL. PMDs initialize this
7931  *   structure in case of error only.
7932  *
7933  * @return
7934  *   0 on success, a negative errno value otherwise and rte_errno is set.
7935  *
7936  * @note: wrapper for driver action_create op callback.
7937  */
7938 static int
7939 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
7940                            struct rte_flow_action_handle *handle,
7941                            struct rte_flow_error *error)
7942 {
7943         static const char err_msg[] = "indirect action destruction unsupported";
7944         struct rte_flow_attr attr = { .transfer = 0 };
7945         const struct mlx5_flow_driver_ops *fops =
7946                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7947
7948         if (!fops->action_destroy) {
7949                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7950                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7951                                    NULL, err_msg);
7952                 return -rte_errno;
7953         }
7954         return fops->action_destroy(dev, handle, error);
7955 }
7956
7957 /* Wrapper for driver action_destroy op callback */
7958 static int
7959 flow_drv_action_update(struct rte_eth_dev *dev,
7960                        struct rte_flow_action_handle *handle,
7961                        const void *update,
7962                        const struct mlx5_flow_driver_ops *fops,
7963                        struct rte_flow_error *error)
7964 {
7965         static const char err_msg[] = "indirect action update unsupported";
7966
7967         if (!fops->action_update) {
7968                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7969                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7970                                    NULL, err_msg);
7971                 return -rte_errno;
7972         }
7973         return fops->action_update(dev, handle, update, error);
7974 }
7975
7976 /* Wrapper for driver action_destroy op callback */
7977 static int
7978 flow_drv_action_query(struct rte_eth_dev *dev,
7979                       const struct rte_flow_action_handle *handle,
7980                       void *data,
7981                       const struct mlx5_flow_driver_ops *fops,
7982                       struct rte_flow_error *error)
7983 {
7984         static const char err_msg[] = "indirect action query unsupported";
7985
7986         if (!fops->action_query) {
7987                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7988                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7989                                    NULL, err_msg);
7990                 return -rte_errno;
7991         }
7992         return fops->action_query(dev, handle, data, error);
7993 }
7994
7995 /**
7996  * Create indirect action for reuse in multiple flow rules.
7997  *
7998  * @param dev
7999  *   Pointer to Ethernet device structure.
8000  * @param conf
8001  *   Pointer to indirect action object configuration.
8002  * @param[in] action
8003  *   Action configuration for indirect action object creation.
8004  * @param[out] error
8005  *   Perform verbose error reporting if not NULL. PMDs initialize this
8006  *   structure in case of error only.
8007  * @return
8008  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8009  */
8010 static struct rte_flow_action_handle *
8011 mlx5_action_handle_create(struct rte_eth_dev *dev,
8012                           const struct rte_flow_indir_action_conf *conf,
8013                           const struct rte_flow_action *action,
8014                           struct rte_flow_error *error)
8015 {
8016         static const char err_msg[] = "indirect action creation unsupported";
8017         struct rte_flow_attr attr = { .transfer = 0 };
8018         const struct mlx5_flow_driver_ops *fops =
8019                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8020
8021         if (flow_drv_action_validate(dev, conf, action, fops, error))
8022                 return NULL;
8023         if (!fops->action_create) {
8024                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8025                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8026                                    NULL, err_msg);
8027                 return NULL;
8028         }
8029         return fops->action_create(dev, conf, action, error);
8030 }
8031
8032 /**
8033  * Updates inplace the indirect action configuration pointed by *handle*
8034  * with the configuration provided as *update* argument.
8035  * The update of the indirect action configuration effects all flow rules
8036  * reusing the action via handle.
8037  *
8038  * @param dev
8039  *   Pointer to Ethernet device structure.
8040  * @param[in] handle
8041  *   Handle for the indirect action to be updated.
8042  * @param[in] update
8043  *   Action specification used to modify the action pointed by handle.
8044  *   *update* could be of same type with the action pointed by the *handle*
8045  *   handle argument, or some other structures like a wrapper, depending on
8046  *   the indirect action type.
8047  * @param[out] error
8048  *   Perform verbose error reporting if not NULL. PMDs initialize this
8049  *   structure in case of error only.
8050  *
8051  * @return
8052  *   0 on success, a negative errno value otherwise and rte_errno is set.
8053  */
8054 static int
8055 mlx5_action_handle_update(struct rte_eth_dev *dev,
8056                 struct rte_flow_action_handle *handle,
8057                 const void *update,
8058                 struct rte_flow_error *error)
8059 {
8060         struct rte_flow_attr attr = { .transfer = 0 };
8061         const struct mlx5_flow_driver_ops *fops =
8062                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8063         int ret;
8064
8065         ret = flow_drv_action_validate(dev, NULL,
8066                         (const struct rte_flow_action *)update, fops, error);
8067         if (ret)
8068                 return ret;
8069         return flow_drv_action_update(dev, handle, update, fops,
8070                                       error);
8071 }
8072
8073 /**
8074  * Query the indirect action by handle.
8075  *
8076  * This function allows retrieving action-specific data such as counters.
8077  * Data is gathered by special action which may be present/referenced in
8078  * more than one flow rule definition.
8079  *
8080  * see @RTE_FLOW_ACTION_TYPE_COUNT
8081  *
8082  * @param dev
8083  *   Pointer to Ethernet device structure.
8084  * @param[in] handle
8085  *   Handle for the indirect action to query.
8086  * @param[in, out] data
8087  *   Pointer to storage for the associated query data type.
8088  * @param[out] error
8089  *   Perform verbose error reporting if not NULL. PMDs initialize this
8090  *   structure in case of error only.
8091  *
8092  * @return
8093  *   0 on success, a negative errno value otherwise and rte_errno is set.
8094  */
8095 static int
8096 mlx5_action_handle_query(struct rte_eth_dev *dev,
8097                          const struct rte_flow_action_handle *handle,
8098                          void *data,
8099                          struct rte_flow_error *error)
8100 {
8101         struct rte_flow_attr attr = { .transfer = 0 };
8102         const struct mlx5_flow_driver_ops *fops =
8103                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8104
8105         return flow_drv_action_query(dev, handle, data, fops, error);
8106 }
8107
8108 /**
8109  * Destroy all indirect actions (shared RSS).
8110  *
8111  * @param dev
8112  *   Pointer to Ethernet device.
8113  *
8114  * @return
8115  *   0 on success, a negative errno value otherwise and rte_errno is set.
8116  */
8117 int
8118 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8119 {
8120         struct rte_flow_error error;
8121         struct mlx5_priv *priv = dev->data->dev_private;
8122         struct mlx5_shared_action_rss *shared_rss;
8123         int ret = 0;
8124         uint32_t idx;
8125
8126         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8127                       priv->rss_shared_actions, idx, shared_rss, next) {
8128                 ret |= mlx5_action_handle_destroy(dev,
8129                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8130         }
8131         return ret;
8132 }
8133
8134 #ifndef HAVE_MLX5DV_DR
8135 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8136 #else
8137 #define MLX5_DOMAIN_SYNC_FLOW \
8138         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8139 #endif
8140
8141 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8142 {
8143         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8144         const struct mlx5_flow_driver_ops *fops;
8145         int ret;
8146         struct rte_flow_attr attr = { .transfer = 0 };
8147
8148         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8149         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8150         if (ret > 0)
8151                 ret = -ret;
8152         return ret;
8153 }
8154
8155 const struct mlx5_flow_tunnel *
8156 mlx5_get_tof(const struct rte_flow_item *item,
8157              const struct rte_flow_action *action,
8158              enum mlx5_tof_rule_type *rule_type)
8159 {
8160         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8161                 if (item->type == (typeof(item->type))
8162                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8163                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8164                         return flow_items_to_tunnel(item);
8165                 }
8166         }
8167         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8168                 if (action->type == (typeof(action->type))
8169                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8170                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8171                         return flow_actions_to_tunnel(action);
8172                 }
8173         }
8174         return NULL;
8175 }
8176
8177 /**
8178  * tunnel offload functionalilty is defined for DV environment only
8179  */
8180 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8181 __extension__
8182 union tunnel_offload_mark {
8183         uint32_t val;
8184         struct {
8185                 uint32_t app_reserve:8;
8186                 uint32_t table_id:15;
8187                 uint32_t transfer:1;
8188                 uint32_t _unused_:8;
8189         };
8190 };
8191
8192 static bool
8193 mlx5_access_tunnel_offload_db
8194         (struct rte_eth_dev *dev,
8195          bool (*match)(struct rte_eth_dev *,
8196                        struct mlx5_flow_tunnel *, const void *),
8197          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8198          void (*miss)(struct rte_eth_dev *, void *),
8199          void *ctx, bool lock_op);
8200
8201 static int
8202 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8203                              struct rte_flow *flow,
8204                              const struct rte_flow_attr *attr,
8205                              const struct rte_flow_action *app_actions,
8206                              uint32_t flow_idx,
8207                              const struct mlx5_flow_tunnel *tunnel,
8208                              struct tunnel_default_miss_ctx *ctx,
8209                              struct rte_flow_error *error)
8210 {
8211         struct mlx5_priv *priv = dev->data->dev_private;
8212         struct mlx5_flow *dev_flow;
8213         struct rte_flow_attr miss_attr = *attr;
8214         const struct rte_flow_item miss_items[2] = {
8215                 {
8216                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8217                         .spec = NULL,
8218                         .last = NULL,
8219                         .mask = NULL
8220                 },
8221                 {
8222                         .type = RTE_FLOW_ITEM_TYPE_END,
8223                         .spec = NULL,
8224                         .last = NULL,
8225                         .mask = NULL
8226                 }
8227         };
8228         union tunnel_offload_mark mark_id;
8229         struct rte_flow_action_mark miss_mark;
8230         struct rte_flow_action miss_actions[3] = {
8231                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8232                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8233         };
8234         const struct rte_flow_action_jump *jump_data;
8235         uint32_t i, flow_table = 0; /* prevent compilation warning */
8236         struct flow_grp_info grp_info = {
8237                 .external = 1,
8238                 .transfer = attr->transfer,
8239                 .fdb_def_rule = !!priv->fdb_def_rule,
8240                 .std_tbl_fix = 0,
8241         };
8242         int ret;
8243
8244         if (!attr->transfer) {
8245                 uint32_t q_size;
8246
8247                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8248                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8249                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8250                                          0, SOCKET_ID_ANY);
8251                 if (!ctx->queue)
8252                         return rte_flow_error_set
8253                                 (error, ENOMEM,
8254                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8255                                 NULL, "invalid default miss RSS");
8256                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8257                 ctx->action_rss.level = 0,
8258                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8259                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8260                 ctx->action_rss.queue_num = priv->reta_idx_n,
8261                 ctx->action_rss.key = priv->rss_conf.rss_key,
8262                 ctx->action_rss.queue = ctx->queue;
8263                 if (!priv->reta_idx_n || !priv->rxqs_n)
8264                         return rte_flow_error_set
8265                                 (error, EINVAL,
8266                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8267                                 NULL, "invalid port configuration");
8268                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8269                         ctx->action_rss.types = 0;
8270                 for (i = 0; i != priv->reta_idx_n; ++i)
8271                         ctx->queue[i] = (*priv->reta_idx)[i];
8272         } else {
8273                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8274                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8275         }
8276         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8277         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8278         jump_data = app_actions->conf;
8279         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8280         miss_attr.group = jump_data->group;
8281         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8282                                        &flow_table, &grp_info, error);
8283         if (ret)
8284                 return rte_flow_error_set(error, EINVAL,
8285                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8286                                           NULL, "invalid tunnel id");
8287         mark_id.app_reserve = 0;
8288         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8289         mark_id.transfer = !!attr->transfer;
8290         mark_id._unused_ = 0;
8291         miss_mark.id = mark_id.val;
8292         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8293                                     miss_items, miss_actions, flow_idx, error);
8294         if (!dev_flow)
8295                 return -rte_errno;
8296         dev_flow->flow = flow;
8297         dev_flow->external = true;
8298         dev_flow->tunnel = tunnel;
8299         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8300         /* Subflow object was created, we must include one in the list. */
8301         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8302                       dev_flow->handle, next);
8303         DRV_LOG(DEBUG,
8304                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8305                 dev->data->port_id, tunnel->app_tunnel.type,
8306                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8307         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8308                                   miss_actions, error);
8309         if (!ret)
8310                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8311                                                   error);
8312
8313         return ret;
8314 }
8315
8316 static const struct mlx5_flow_tbl_data_entry  *
8317 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8318 {
8319         struct mlx5_priv *priv = dev->data->dev_private;
8320         struct mlx5_dev_ctx_shared *sh = priv->sh;
8321         struct mlx5_hlist_entry *he;
8322         union tunnel_offload_mark mbits = { .val = mark };
8323         union mlx5_flow_tbl_key table_key = {
8324                 {
8325                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8326                         .id = 0,
8327                         .reserved = 0,
8328                         .dummy = 0,
8329                         .is_fdb = !!mbits.transfer,
8330                         .is_egress = 0,
8331                 }
8332         };
8333         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
8334         return he ?
8335                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8336 }
8337
8338 static void
8339 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
8340                                    struct mlx5_hlist_entry *entry)
8341 {
8342         struct mlx5_dev_ctx_shared *sh = list->ctx;
8343         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8344
8345         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8346                         tunnel_flow_tbl_to_id(tte->flow_table));
8347         mlx5_free(tte);
8348 }
8349
8350 static int
8351 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
8352                                   struct mlx5_hlist_entry *entry,
8353                                   uint64_t key, void *cb_ctx __rte_unused)
8354 {
8355         union tunnel_tbl_key tbl = {
8356                 .val = key,
8357         };
8358         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8359
8360         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8361 }
8362
8363 static struct mlx5_hlist_entry *
8364 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
8365                                    void *ctx __rte_unused)
8366 {
8367         struct mlx5_dev_ctx_shared *sh = list->ctx;
8368         struct tunnel_tbl_entry *tte;
8369         union tunnel_tbl_key tbl = {
8370                 .val = key,
8371         };
8372
8373         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8374                           sizeof(*tte), 0,
8375                           SOCKET_ID_ANY);
8376         if (!tte)
8377                 goto err;
8378         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8379                           &tte->flow_table);
8380         if (tte->flow_table >= MLX5_MAX_TABLES) {
8381                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8382                         tte->flow_table);
8383                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8384                                 tte->flow_table);
8385                 goto err;
8386         } else if (!tte->flow_table) {
8387                 goto err;
8388         }
8389         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8390         tte->tunnel_id = tbl.tunnel_id;
8391         tte->group = tbl.group;
8392         return &tte->hash;
8393 err:
8394         if (tte)
8395                 mlx5_free(tte);
8396         return NULL;
8397 }
8398
8399 static uint32_t
8400 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8401                                 const struct mlx5_flow_tunnel *tunnel,
8402                                 uint32_t group, uint32_t *table,
8403                                 struct rte_flow_error *error)
8404 {
8405         struct mlx5_hlist_entry *he;
8406         struct tunnel_tbl_entry *tte;
8407         union tunnel_tbl_key key = {
8408                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8409                 .group = group
8410         };
8411         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8412         struct mlx5_hlist *group_hash;
8413
8414         group_hash = tunnel ? tunnel->groups : thub->groups;
8415         he = mlx5_hlist_register(group_hash, key.val, NULL);
8416         if (!he)
8417                 return rte_flow_error_set(error, EINVAL,
8418                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8419                                           NULL,
8420                                           "tunnel group index not supported");
8421         tte = container_of(he, typeof(*tte), hash);
8422         *table = tte->flow_table;
8423         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8424                 dev->data->port_id, key.tunnel_id, group, *table);
8425         return 0;
8426 }
8427
8428 static void
8429 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8430                       struct mlx5_flow_tunnel *tunnel)
8431 {
8432         struct mlx5_priv *priv = dev->data->dev_private;
8433         struct mlx5_indexed_pool *ipool;
8434
8435         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8436                 dev->data->port_id, tunnel->tunnel_id);
8437         LIST_REMOVE(tunnel, chain);
8438         mlx5_hlist_destroy(tunnel->groups);
8439         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8440         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8441 }
8442
8443 static bool
8444 mlx5_access_tunnel_offload_db
8445         (struct rte_eth_dev *dev,
8446          bool (*match)(struct rte_eth_dev *,
8447                        struct mlx5_flow_tunnel *, const void *),
8448          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8449          void (*miss)(struct rte_eth_dev *, void *),
8450          void *ctx, bool lock_op)
8451 {
8452         bool verdict = false;
8453         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8454         struct mlx5_flow_tunnel *tunnel;
8455
8456         rte_spinlock_lock(&thub->sl);
8457         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8458                 verdict = match(dev, tunnel, (const void *)ctx);
8459                 if (verdict)
8460                         break;
8461         }
8462         if (!lock_op)
8463                 rte_spinlock_unlock(&thub->sl);
8464         if (verdict && hit)
8465                 hit(dev, tunnel, ctx);
8466         if (!verdict && miss)
8467                 miss(dev, ctx);
8468         if (lock_op)
8469                 rte_spinlock_unlock(&thub->sl);
8470
8471         return verdict;
8472 }
8473
8474 struct tunnel_db_find_tunnel_id_ctx {
8475         uint32_t tunnel_id;
8476         struct mlx5_flow_tunnel *tunnel;
8477 };
8478
8479 static bool
8480 find_tunnel_id_match(struct rte_eth_dev *dev,
8481                      struct mlx5_flow_tunnel *tunnel, const void *x)
8482 {
8483         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8484
8485         RTE_SET_USED(dev);
8486         return tunnel->tunnel_id == ctx->tunnel_id;
8487 }
8488
8489 static void
8490 find_tunnel_id_hit(struct rte_eth_dev *dev,
8491                    struct mlx5_flow_tunnel *tunnel, void *x)
8492 {
8493         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8494         RTE_SET_USED(dev);
8495         ctx->tunnel = tunnel;
8496 }
8497
8498 static struct mlx5_flow_tunnel *
8499 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8500 {
8501         struct tunnel_db_find_tunnel_id_ctx ctx = {
8502                 .tunnel_id = id,
8503         };
8504
8505         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8506                                       find_tunnel_id_hit, NULL, &ctx, true);
8507
8508         return ctx.tunnel;
8509 }
8510
8511 static struct mlx5_flow_tunnel *
8512 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8513                           const struct rte_flow_tunnel *app_tunnel)
8514 {
8515         struct mlx5_priv *priv = dev->data->dev_private;
8516         struct mlx5_indexed_pool *ipool;
8517         struct mlx5_flow_tunnel *tunnel;
8518         uint32_t id;
8519
8520         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8521         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8522         if (!tunnel)
8523                 return NULL;
8524         if (id >= MLX5_MAX_TUNNELS) {
8525                 mlx5_ipool_free(ipool, id);
8526                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8527                 return NULL;
8528         }
8529         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
8530                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8531                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8532                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
8533         if (!tunnel->groups) {
8534                 mlx5_ipool_free(ipool, id);
8535                 return NULL;
8536         }
8537         tunnel->groups->ctx = priv->sh;
8538         /* initiate new PMD tunnel */
8539         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8540         tunnel->tunnel_id = id;
8541         tunnel->action.type = (typeof(tunnel->action.type))
8542                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8543         tunnel->action.conf = tunnel;
8544         tunnel->item.type = (typeof(tunnel->item.type))
8545                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8546         tunnel->item.spec = tunnel;
8547         tunnel->item.last = NULL;
8548         tunnel->item.mask = NULL;
8549
8550         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8551                 dev->data->port_id, tunnel->tunnel_id);
8552
8553         return tunnel;
8554 }
8555
8556 struct tunnel_db_get_tunnel_ctx {
8557         const struct rte_flow_tunnel *app_tunnel;
8558         struct mlx5_flow_tunnel *tunnel;
8559 };
8560
8561 static bool get_tunnel_match(struct rte_eth_dev *dev,
8562                              struct mlx5_flow_tunnel *tunnel, const void *x)
8563 {
8564         const struct tunnel_db_get_tunnel_ctx *ctx = x;
8565
8566         RTE_SET_USED(dev);
8567         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
8568                        sizeof(*ctx->app_tunnel));
8569 }
8570
8571 static void get_tunnel_hit(struct rte_eth_dev *dev,
8572                            struct mlx5_flow_tunnel *tunnel, void *x)
8573 {
8574         /* called under tunnel spinlock protection */
8575         struct tunnel_db_get_tunnel_ctx *ctx = x;
8576
8577         RTE_SET_USED(dev);
8578         tunnel->refctn++;
8579         ctx->tunnel = tunnel;
8580 }
8581
8582 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
8583 {
8584         /* called under tunnel spinlock protection */
8585         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8586         struct tunnel_db_get_tunnel_ctx *ctx = x;
8587
8588         rte_spinlock_unlock(&thub->sl);
8589         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
8590         rte_spinlock_lock(&thub->sl);
8591         if (ctx->tunnel) {
8592                 ctx->tunnel->refctn = 1;
8593                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
8594         }
8595 }
8596
8597
8598 static int
8599 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8600                      const struct rte_flow_tunnel *app_tunnel,
8601                      struct mlx5_flow_tunnel **tunnel)
8602 {
8603         struct tunnel_db_get_tunnel_ctx ctx = {
8604                 .app_tunnel = app_tunnel,
8605         };
8606
8607         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
8608                                       get_tunnel_miss, &ctx, true);
8609         *tunnel = ctx.tunnel;
8610         return ctx.tunnel ? 0 : -ENOMEM;
8611 }
8612
8613 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8614 {
8615         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8616
8617         if (!thub)
8618                 return;
8619         if (!LIST_EMPTY(&thub->tunnels))
8620                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
8621         mlx5_hlist_destroy(thub->groups);
8622         mlx5_free(thub);
8623 }
8624
8625 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8626 {
8627         int err;
8628         struct mlx5_flow_tunnel_hub *thub;
8629
8630         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8631                            0, SOCKET_ID_ANY);
8632         if (!thub)
8633                 return -ENOMEM;
8634         LIST_INIT(&thub->tunnels);
8635         rte_spinlock_init(&thub->sl);
8636         thub->groups = mlx5_hlist_create("flow groups",
8637                                          rte_align32pow2(MLX5_MAX_TABLES), 0,
8638                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
8639                                          mlx5_flow_tunnel_grp2tbl_match_cb,
8640                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
8641         if (!thub->groups) {
8642                 err = -rte_errno;
8643                 goto err;
8644         }
8645         thub->groups->ctx = sh;
8646         sh->tunnel_hub = thub;
8647
8648         return 0;
8649
8650 err:
8651         if (thub->groups)
8652                 mlx5_hlist_destroy(thub->groups);
8653         if (thub)
8654                 mlx5_free(thub);
8655         return err;
8656 }
8657
8658 static inline bool
8659 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
8660                           struct rte_flow_tunnel *tunnel,
8661                           const char *err_msg)
8662 {
8663         err_msg = NULL;
8664         if (!is_tunnel_offload_active(dev)) {
8665                 err_msg = "tunnel offload was not activated";
8666                 goto out;
8667         } else if (!tunnel) {
8668                 err_msg = "no application tunnel";
8669                 goto out;
8670         }
8671
8672         switch (tunnel->type) {
8673         default:
8674                 err_msg = "unsupported tunnel type";
8675                 goto out;
8676         case RTE_FLOW_ITEM_TYPE_VXLAN:
8677                 break;
8678         }
8679
8680 out:
8681         return !err_msg;
8682 }
8683
8684 static int
8685 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
8686                     struct rte_flow_tunnel *app_tunnel,
8687                     struct rte_flow_action **actions,
8688                     uint32_t *num_of_actions,
8689                     struct rte_flow_error *error)
8690 {
8691         int ret;
8692         struct mlx5_flow_tunnel *tunnel;
8693         const char *err_msg = NULL;
8694         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8695
8696         if (!verdict)
8697                 return rte_flow_error_set(error, EINVAL,
8698                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8699                                           err_msg);
8700         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8701         if (ret < 0) {
8702                 return rte_flow_error_set(error, ret,
8703                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8704                                           "failed to initialize pmd tunnel");
8705         }
8706         *actions = &tunnel->action;
8707         *num_of_actions = 1;
8708         return 0;
8709 }
8710
8711 static int
8712 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
8713                        struct rte_flow_tunnel *app_tunnel,
8714                        struct rte_flow_item **items,
8715                        uint32_t *num_of_items,
8716                        struct rte_flow_error *error)
8717 {
8718         int ret;
8719         struct mlx5_flow_tunnel *tunnel;
8720         const char *err_msg = NULL;
8721         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8722
8723         if (!verdict)
8724                 return rte_flow_error_set(error, EINVAL,
8725                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8726                                           err_msg);
8727         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8728         if (ret < 0) {
8729                 return rte_flow_error_set(error, ret,
8730                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8731                                           "failed to initialize pmd tunnel");
8732         }
8733         *items = &tunnel->item;
8734         *num_of_items = 1;
8735         return 0;
8736 }
8737
8738 struct tunnel_db_element_release_ctx {
8739         struct rte_flow_item *items;
8740         struct rte_flow_action *actions;
8741         uint32_t num_elements;
8742         struct rte_flow_error *error;
8743         int ret;
8744 };
8745
8746 static bool
8747 tunnel_element_release_match(struct rte_eth_dev *dev,
8748                              struct mlx5_flow_tunnel *tunnel, const void *x)
8749 {
8750         const struct tunnel_db_element_release_ctx *ctx = x;
8751
8752         RTE_SET_USED(dev);
8753         if (ctx->num_elements != 1)
8754                 return false;
8755         else if (ctx->items)
8756                 return ctx->items == &tunnel->item;
8757         else if (ctx->actions)
8758                 return ctx->actions == &tunnel->action;
8759
8760         return false;
8761 }
8762
8763 static void
8764 tunnel_element_release_hit(struct rte_eth_dev *dev,
8765                            struct mlx5_flow_tunnel *tunnel, void *x)
8766 {
8767         struct tunnel_db_element_release_ctx *ctx = x;
8768         ctx->ret = 0;
8769         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
8770                 mlx5_flow_tunnel_free(dev, tunnel);
8771 }
8772
8773 static void
8774 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
8775 {
8776         struct tunnel_db_element_release_ctx *ctx = x;
8777         RTE_SET_USED(dev);
8778         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
8779                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8780                                       "invalid argument");
8781 }
8782
8783 static int
8784 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
8785                        struct rte_flow_item *pmd_items,
8786                        uint32_t num_items, struct rte_flow_error *err)
8787 {
8788         struct tunnel_db_element_release_ctx ctx = {
8789                 .items = pmd_items,
8790                 .actions = NULL,
8791                 .num_elements = num_items,
8792                 .error = err,
8793         };
8794
8795         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8796                                       tunnel_element_release_hit,
8797                                       tunnel_element_release_miss, &ctx, false);
8798
8799         return ctx.ret;
8800 }
8801
8802 static int
8803 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
8804                          struct rte_flow_action *pmd_actions,
8805                          uint32_t num_actions, struct rte_flow_error *err)
8806 {
8807         struct tunnel_db_element_release_ctx ctx = {
8808                 .items = NULL,
8809                 .actions = pmd_actions,
8810                 .num_elements = num_actions,
8811                 .error = err,
8812         };
8813
8814         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8815                                       tunnel_element_release_hit,
8816                                       tunnel_element_release_miss, &ctx, false);
8817
8818         return ctx.ret;
8819 }
8820
8821 static int
8822 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
8823                                   struct rte_mbuf *m,
8824                                   struct rte_flow_restore_info *info,
8825                                   struct rte_flow_error *err)
8826 {
8827         uint64_t ol_flags = m->ol_flags;
8828         const struct mlx5_flow_tbl_data_entry *tble;
8829         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
8830
8831         if (!is_tunnel_offload_active(dev)) {
8832                 info->flags = 0;
8833                 return 0;
8834         }
8835
8836         if ((ol_flags & mask) != mask)
8837                 goto err;
8838         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
8839         if (!tble) {
8840                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
8841                         dev->data->port_id, m->hash.fdir.hi);
8842                 goto err;
8843         }
8844         MLX5_ASSERT(tble->tunnel);
8845         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
8846         info->group_id = tble->group_id;
8847         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
8848                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
8849                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
8850
8851         return 0;
8852
8853 err:
8854         return rte_flow_error_set(err, EINVAL,
8855                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8856                                   "failed to get restore info");
8857 }
8858
8859 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
8860 static int
8861 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
8862                            __rte_unused struct rte_flow_tunnel *app_tunnel,
8863                            __rte_unused struct rte_flow_action **actions,
8864                            __rte_unused uint32_t *num_of_actions,
8865                            __rte_unused struct rte_flow_error *error)
8866 {
8867         return -ENOTSUP;
8868 }
8869
8870 static int
8871 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
8872                        __rte_unused struct rte_flow_tunnel *app_tunnel,
8873                        __rte_unused struct rte_flow_item **items,
8874                        __rte_unused uint32_t *num_of_items,
8875                        __rte_unused struct rte_flow_error *error)
8876 {
8877         return -ENOTSUP;
8878 }
8879
8880 static int
8881 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
8882                               __rte_unused struct rte_flow_item *pmd_items,
8883                               __rte_unused uint32_t num_items,
8884                               __rte_unused struct rte_flow_error *err)
8885 {
8886         return -ENOTSUP;
8887 }
8888
8889 static int
8890 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
8891                                 __rte_unused struct rte_flow_action *pmd_action,
8892                                 __rte_unused uint32_t num_actions,
8893                                 __rte_unused struct rte_flow_error *err)
8894 {
8895         return -ENOTSUP;
8896 }
8897
8898 static int
8899 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
8900                                   __rte_unused struct rte_mbuf *m,
8901                                   __rte_unused struct rte_flow_restore_info *i,
8902                                   __rte_unused struct rte_flow_error *err)
8903 {
8904         return -ENOTSUP;
8905 }
8906
8907 static int
8908 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
8909                              __rte_unused struct rte_flow *flow,
8910                              __rte_unused const struct rte_flow_attr *attr,
8911                              __rte_unused const struct rte_flow_action *actions,
8912                              __rte_unused uint32_t flow_idx,
8913                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
8914                              __rte_unused struct tunnel_default_miss_ctx *ctx,
8915                              __rte_unused struct rte_flow_error *error)
8916 {
8917         return -ENOTSUP;
8918 }
8919
8920 static struct mlx5_flow_tunnel *
8921 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
8922                     __rte_unused uint32_t id)
8923 {
8924         return NULL;
8925 }
8926
8927 static void
8928 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
8929                       __rte_unused struct mlx5_flow_tunnel *tunnel)
8930 {
8931 }
8932
8933 static uint32_t
8934 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
8935                                 __rte_unused const struct mlx5_flow_tunnel *t,
8936                                 __rte_unused uint32_t group,
8937                                 __rte_unused uint32_t *table,
8938                                 struct rte_flow_error *error)
8939 {
8940         return rte_flow_error_set(error, ENOTSUP,
8941                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8942                                   "tunnel offload requires DV support");
8943 }
8944
8945 void
8946 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
8947                         __rte_unused  uint16_t port_id)
8948 {
8949 }
8950 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */