drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              struct tunnel_default_miss_ctx *ctx,
  54                              struct rte_flow_error *error);
  55 static struct mlx5_flow_tunnel *
  56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  57 static void
  58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  59 static uint32_t
  60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  61                                 const struct mlx5_flow_tunnel *tunnel,
  62                                 uint32_t group, uint32_t *table,
  63                                 struct rte_flow_error *error);
  64
  65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  66 static void mlx5_flow_pop_thread_workspace(void);
  67
  68
  69 /** Device flow drivers. */
  70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  71
  72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  73
  74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  75         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  77         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  78 #endif
  79         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  80         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  81 };
  82
  83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  85         (const int []){ \
  86                 __VA_ARGS__, 0, \
  87         }
  88
  89 /** Node object of input graph for mlx5_flow_expand_rss(). */
  90 struct mlx5_flow_expand_node {
  91         const int *const next;
  92         /**<
  93          * List of next node indexes. Index 0 is interpreted as a terminator.
  94          */
  95         const enum rte_flow_item_type type;
  96         /**< Pattern item type of current node. */
  97         uint64_t rss_types;
  98         /**<
  99          * RSS types bit-field associated with this node
 100          * (see ETH_RSS_* definitions).
 101          */
 102 };
 103
 104 /** Object returned by mlx5_flow_expand_rss(). */
 105 struct mlx5_flow_expand_rss {
 106         uint32_t entries;
 107         /**< Number of entries @p patterns and @p priorities. */
 108         struct {
 109                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 110                 uint32_t priority; /**< Priority offset for each expansion. */
 111         } entry[];
 112 };
 113
 114 static enum rte_flow_item_type
 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 116 {
 117         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 118         uint16_t ether_type = 0;
 119         uint16_t ether_type_m;
 120         uint8_t ip_next_proto = 0;
 121         uint8_t ip_next_proto_m;
 122
 123         if (item == NULL || item->spec == NULL)
 124                 return ret;
 125         switch (item->type) {
 126         case RTE_FLOW_ITEM_TYPE_ETH:
 127                 if (item->mask)
 128                         ether_type_m = ((const struct rte_flow_item_eth *)
 129                                                 (item->mask))->type;
 130                 else
 131                         ether_type_m = rte_flow_item_eth_mask.type;
 132                 if (ether_type_m != RTE_BE16(0xFFFF))
 133                         break;
 134                 ether_type = ((const struct rte_flow_item_eth *)
 135                                 (item->spec))->type;
 136                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 139                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 140                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 141                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 142                 else
 143                         ret = RTE_FLOW_ITEM_TYPE_END;
 144                 break;
 145         case RTE_FLOW_ITEM_TYPE_VLAN:
 146                 if (item->mask)
 147                         ether_type_m = ((const struct rte_flow_item_vlan *)
 148                                                 (item->mask))->inner_type;
 149                 else
 150                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 151                 if (ether_type_m != RTE_BE16(0xFFFF))
 152                         break;
 153                 ether_type = ((const struct rte_flow_item_vlan *)
 154                                 (item->spec))->inner_type;
 155                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 156                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 157                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 159                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 160                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV4:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 167                                         (item->mask))->hdr.next_proto_id;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 174                                 (item->spec))->hdr.next_proto_id;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         case RTE_FLOW_ITEM_TYPE_IPV6:
 187                 if (item->mask)
 188                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 189                                                 (item->mask))->hdr.proto;
 190                 else
 191                         ip_next_proto_m =
 192                                 rte_flow_item_ipv6_mask.hdr.proto;
 193                 if (ip_next_proto_m != 0xFF)
 194                         break;
 195                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 196                                 (item->spec))->hdr.proto;
 197                 if (ip_next_proto == IPPROTO_UDP)
 198                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 199                 else if (ip_next_proto == IPPROTO_TCP)
 200                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 201                 else if (ip_next_proto == IPPROTO_IP)
 202                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 203                 else if (ip_next_proto == IPPROTO_IPV6)
 204                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 205                 else
 206                         ret = RTE_FLOW_ITEM_TYPE_END;
 207                 break;
 208         default:
 209                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 210                 break;
 211         }
 212         return ret;
 213 }
 214
 215 /**
 216  * Expand RSS flows into several possible flows according to the RSS hash
 217  * fields requested and the driver capabilities.
 218  *
 219  * @param[out] buf
 220  *   Buffer to store the result expansion.
 221  * @param[in] size
 222  *   Buffer size in bytes. If 0, @p buf can be NULL.
 223  * @param[in] pattern
 224  *   User flow pattern.
 225  * @param[in] types
 226  *   RSS types to expand (see ETH_RSS_* definitions).
 227  * @param[in] graph
 228  *   Input graph to expand @p pattern according to @p types.
 229  * @param[in] graph_root_index
 230  *   Index of root node in @p graph, typically 0.
 231  *
 232  * @return
 233  *   A positive value representing the size of @p buf in bytes regardless of
 234  *   @p size on success, a negative errno value otherwise and rte_errno is
 235  *   set, the following errors are defined:
 236  *
 237  *   -E2BIG: graph-depth @p graph is too deep.
 238  */
 239 static int
 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 241                      const struct rte_flow_item *pattern, uint64_t types,
 242                      const struct mlx5_flow_expand_node graph[],
 243                      int graph_root_index)
 244 {
 245         const int elt_n = 8;
 246         const struct rte_flow_item *item;
 247         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 248         const int *next_node;
 249         const int *stack[elt_n];
 250         int stack_pos = 0;
 251         struct rte_flow_item flow_items[elt_n];
 252         unsigned int i;
 253         size_t lsize;
 254         size_t user_pattern_size = 0;
 255         void *addr = NULL;
 256         const struct mlx5_flow_expand_node *next = NULL;
 257         struct rte_flow_item missed_item;
 258         int missed = 0;
 259         int elt = 0;
 260         const struct rte_flow_item *last_item = NULL;
 261
 262         memset(&missed_item, 0, sizeof(missed_item));
 263         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 264                 elt_n * sizeof(buf->entry[0]);
 265         if (lsize <= size) {
 266                 buf->entry[0].priority = 0;
 267                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 268                 buf->entries = 0;
 269                 addr = buf->entry[0].pattern;
 270         }
 271         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 272                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 273                         last_item = item;
 274                 for (i = 0; node->next && node->next[i]; ++i) {
 275                         next = &graph[node->next[i]];
 276                         if (next->type == item->type)
 277                                 break;
 278                 }
 279                 if (next)
 280                         node = next;
 281                 user_pattern_size += sizeof(*item);
 282         }
 283         user_pattern_size += sizeof(*item); /* Handle END item. */
 284         lsize += user_pattern_size;
 285         /* Copy the user pattern in the first entry of the buffer. */
 286         if (lsize <= size) {
 287                 rte_memcpy(addr, pattern, user_pattern_size);
 288                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 289                 buf->entries = 1;
 290         }
 291         /* Start expanding. */
 292         memset(flow_items, 0, sizeof(flow_items));
 293         user_pattern_size -= sizeof(*item);
 294         /*
 295          * Check if the last valid item has spec set, need complete pattern,
 296          * and the pattern can be used for expansion.
 297          */
 298         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 299         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 300                 /* Item type END indicates expansion is not required. */
 301                 return lsize;
 302         }
 303         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 304                 next = NULL;
 305                 missed = 1;
 306                 for (i = 0; node->next && node->next[i]; ++i) {
 307                         next = &graph[node->next[i]];
 308                         if (next->type == missed_item.type) {
 309                                 flow_items[0].type = missed_item.type;
 310                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 311                                 break;
 312                         }
 313                         next = NULL;
 314                 }
 315         }
 316         if (next && missed) {
 317                 elt = 2; /* missed item + item end. */
 318                 node = next;
 319                 lsize += elt * sizeof(*item) + user_pattern_size;
 320                 if ((node->rss_types & types) && lsize <= size) {
 321                         buf->entry[buf->entries].priority = 1;
 322                         buf->entry[buf->entries].pattern = addr;
 323                         buf->entries++;
 324                         rte_memcpy(addr, buf->entry[0].pattern,
 325                                    user_pattern_size);
 326                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 327                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 328                         addr = (void *)(((uintptr_t)addr) +
 329                                         elt * sizeof(*item));
 330                 }
 331         }
 332         memset(flow_items, 0, sizeof(flow_items));
 333         next_node = node->next;
 334         stack[stack_pos] = next_node;
 335         node = next_node ? &graph[*next_node] : NULL;
 336         while (node) {
 337                 flow_items[stack_pos].type = node->type;
 338                 if (node->rss_types & types) {
 339                         /*
 340                          * compute the number of items to copy from the
 341                          * expansion and copy it.
 342                          * When the stack_pos is 0, there are 1 element in it,
 343                          * plus the addition END item.
 344                          */
 345                         elt = stack_pos + 2;
 346                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 347                         lsize += elt * sizeof(*item) + user_pattern_size;
 348                         if (lsize <= size) {
 349                                 size_t n = elt * sizeof(*item);
 350
 351                                 buf->entry[buf->entries].priority =
 352                                         stack_pos + 1 + missed;
 353                                 buf->entry[buf->entries].pattern = addr;
 354                                 buf->entries++;
 355                                 rte_memcpy(addr, buf->entry[0].pattern,
 356                                            user_pattern_size);
 357                                 addr = (void *)(((uintptr_t)addr) +
 358                                                 user_pattern_size);
 359                                 rte_memcpy(addr, &missed_item,
 360                                            missed * sizeof(*item));
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                         missed * sizeof(*item));
 363                                 rte_memcpy(addr, flow_items, n);
 364                                 addr = (void *)(((uintptr_t)addr) + n);
 365                         }
 366                 }
 367                 /* Go deeper. */
 368                 if (node->next) {
 369                         next_node = node->next;
 370                         if (stack_pos++ == elt_n) {
 371                                 rte_errno = E2BIG;
 372                                 return -rte_errno;
 373                         }
 374                         stack[stack_pos] = next_node;
 375                 } else if (*(next_node + 1)) {
 376                         /* Follow up with the next possibility. */
 377                         ++next_node;
 378                 } else {
 379                         /* Move to the next path. */
 380                         if (stack_pos)
 381                                 next_node = stack[--stack_pos];
 382                         next_node++;
 383                         stack[stack_pos] = next_node;
 384                 }
 385                 node = *next_node ? &graph[*next_node] : NULL;
 386         };
 387         /* no expanded flows but we have missed item, create one rule for it */
 388         if (buf->entries == 1 && missed != 0) {
 389                 elt = 2;
 390                 lsize += elt * sizeof(*item) + user_pattern_size;
 391                 if (lsize <= size) {
 392                         buf->entry[buf->entries].priority = 1;
 393                         buf->entry[buf->entries].pattern = addr;
 394                         buf->entries++;
 395                         flow_items[0].type = missed_item.type;
 396                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 397                         rte_memcpy(addr, buf->entry[0].pattern,
 398                                    user_pattern_size);
 399                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 400                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 401                 }
 402         }
 403         return lsize;
 404 }
 405
 406 enum mlx5_expansion {
 407         MLX5_EXPANSION_ROOT,
 408         MLX5_EXPANSION_ROOT_OUTER,
 409         MLX5_EXPANSION_ROOT_ETH_VLAN,
 410         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 411         MLX5_EXPANSION_OUTER_ETH,
 412         MLX5_EXPANSION_OUTER_ETH_VLAN,
 413         MLX5_EXPANSION_OUTER_VLAN,
 414         MLX5_EXPANSION_OUTER_IPV4,
 415         MLX5_EXPANSION_OUTER_IPV4_UDP,
 416         MLX5_EXPANSION_OUTER_IPV4_TCP,
 417         MLX5_EXPANSION_OUTER_IPV6,
 418         MLX5_EXPANSION_OUTER_IPV6_UDP,
 419         MLX5_EXPANSION_OUTER_IPV6_TCP,
 420         MLX5_EXPANSION_VXLAN,
 421         MLX5_EXPANSION_VXLAN_GPE,
 422         MLX5_EXPANSION_GRE,
 423         MLX5_EXPANSION_MPLS,
 424         MLX5_EXPANSION_ETH,
 425         MLX5_EXPANSION_ETH_VLAN,
 426         MLX5_EXPANSION_VLAN,
 427         MLX5_EXPANSION_IPV4,
 428         MLX5_EXPANSION_IPV4_UDP,
 429         MLX5_EXPANSION_IPV4_TCP,
 430         MLX5_EXPANSION_IPV6,
 431         MLX5_EXPANSION_IPV6_UDP,
 432         MLX5_EXPANSION_IPV6_TCP,
 433 };
 434
 435 /** Supported expansion of items. */
 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 437         [MLX5_EXPANSION_ROOT] = {
 438                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 439                                                   MLX5_EXPANSION_IPV4,
 440                                                   MLX5_EXPANSION_IPV6),
 441                 .type = RTE_FLOW_ITEM_TYPE_END,
 442         },
 443         [MLX5_EXPANSION_ROOT_OUTER] = {
 444                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 445                                                   MLX5_EXPANSION_OUTER_IPV4,
 446                                                   MLX5_EXPANSION_OUTER_IPV6),
 447                 .type = RTE_FLOW_ITEM_TYPE_END,
 448         },
 449         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 450                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 451                 .type = RTE_FLOW_ITEM_TYPE_END,
 452         },
 453         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 454                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 455                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_END,
 457         },
 458         [MLX5_EXPANSION_OUTER_ETH] = {
 459                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 460                                                   MLX5_EXPANSION_OUTER_IPV6,
 461                                                   MLX5_EXPANSION_MPLS),
 462                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 463                 .rss_types = 0,
 464         },
 465         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 466                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 467                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 468                 .rss_types = 0,
 469         },
 470         [MLX5_EXPANSION_OUTER_VLAN] = {
 471                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 472                                                   MLX5_EXPANSION_OUTER_IPV6),
 473                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 477                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 478                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 479                          MLX5_EXPANSION_GRE,
 480                          MLX5_EXPANSION_IPV4,
 481                          MLX5_EXPANSION_IPV6),
 482                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 483                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 484                         ETH_RSS_NONFRAG_IPV4_OTHER,
 485         },
 486         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 487                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 488                                                   MLX5_EXPANSION_VXLAN_GPE),
 489                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 490                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 493                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 494                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 498                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 499                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 500                          MLX5_EXPANSION_IPV4,
 501                          MLX5_EXPANSION_IPV6),
 502                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 503                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 504                         ETH_RSS_NONFRAG_IPV6_OTHER,
 505         },
 506         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 508                                                   MLX5_EXPANSION_VXLAN_GPE),
 509                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 510                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 511         },
 512         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 513                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 514                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 515         },
 516         [MLX5_EXPANSION_VXLAN] = {
 517                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 518                                                   MLX5_EXPANSION_IPV4,
 519                                                   MLX5_EXPANSION_IPV6),
 520                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 521         },
 522         [MLX5_EXPANSION_VXLAN_GPE] = {
 523                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 524                                                   MLX5_EXPANSION_IPV4,
 525                                                   MLX5_EXPANSION_IPV6),
 526                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 527         },
 528         [MLX5_EXPANSION_GRE] = {
 529                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 530                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 531         },
 532         [MLX5_EXPANSION_MPLS] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 534                                                   MLX5_EXPANSION_IPV6),
 535                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 536         },
 537         [MLX5_EXPANSION_ETH] = {
 538                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 539                                                   MLX5_EXPANSION_IPV6),
 540                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 541         },
 542         [MLX5_EXPANSION_ETH_VLAN] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 544                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 545         },
 546         [MLX5_EXPANSION_VLAN] = {
 547                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 548                                                   MLX5_EXPANSION_IPV6),
 549                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 550         },
 551         [MLX5_EXPANSION_IPV4] = {
 552                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 553                                                   MLX5_EXPANSION_IPV4_TCP),
 554                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 555                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 556                         ETH_RSS_NONFRAG_IPV4_OTHER,
 557         },
 558         [MLX5_EXPANSION_IPV4_UDP] = {
 559                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 560                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 561         },
 562         [MLX5_EXPANSION_IPV4_TCP] = {
 563                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 564                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 565         },
 566         [MLX5_EXPANSION_IPV6] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 568                                                   MLX5_EXPANSION_IPV6_TCP),
 569                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 570                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 571                         ETH_RSS_NONFRAG_IPV6_OTHER,
 572         },
 573         [MLX5_EXPANSION_IPV6_UDP] = {
 574                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 575                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 576         },
 577         [MLX5_EXPANSION_IPV6_TCP] = {
 578                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 579                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 580         },
 581 };
 582
 583 static struct rte_flow_shared_action *
 584 mlx5_shared_action_create(struct rte_eth_dev *dev,
 585                           const struct rte_flow_shared_action_conf *conf,
 586                           const struct rte_flow_action *action,
 587                           struct rte_flow_error *error);
 588 static int mlx5_shared_action_destroy
 589                                 (struct rte_eth_dev *dev,
 590                                  struct rte_flow_shared_action *shared_action,
 591                                  struct rte_flow_error *error);
 592 static int mlx5_shared_action_update
 593                                 (struct rte_eth_dev *dev,
 594                                  struct rte_flow_shared_action *shared_action,
 595                                  const struct rte_flow_action *action,
 596                                  struct rte_flow_error *error);
 597 static int mlx5_shared_action_query
 598                                 (struct rte_eth_dev *dev,
 599                                  const struct rte_flow_shared_action *action,
 600                                  void *data,
 601                                  struct rte_flow_error *error);
 602 static int
 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 604                     struct rte_flow_tunnel *app_tunnel,
 605                     struct rte_flow_action **actions,
 606                     uint32_t *num_of_actions,
 607                     struct rte_flow_error *error);
 608 static int
 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 610                        struct rte_flow_tunnel *app_tunnel,
 611                        struct rte_flow_item **items,
 612                        uint32_t *num_of_items,
 613                        struct rte_flow_error *error);
 614 static int
 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 616                               struct rte_flow_item *pmd_items,
 617                               uint32_t num_items, struct rte_flow_error *err);
 618 static int
 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 620                                 struct rte_flow_action *pmd_actions,
 621                                 uint32_t num_actions,
 622                                 struct rte_flow_error *err);
 623 static int
 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 625                                   struct rte_mbuf *m,
 626                                   struct rte_flow_restore_info *info,
 627                                   struct rte_flow_error *err);
 628
 629 static const struct rte_flow_ops mlx5_flow_ops = {
 630         .validate = mlx5_flow_validate,
 631         .create = mlx5_flow_create,
 632         .destroy = mlx5_flow_destroy,
 633         .flush = mlx5_flow_flush,
 634         .isolate = mlx5_flow_isolate,
 635         .query = mlx5_flow_query,
 636         .dev_dump = mlx5_flow_dev_dump,
 637         .get_aged_flows = mlx5_flow_get_aged_flows,
 638         .shared_action_create = mlx5_shared_action_create,
 639         .shared_action_destroy = mlx5_shared_action_destroy,
 640         .shared_action_update = mlx5_shared_action_update,
 641         .shared_action_query = mlx5_shared_action_query,
 642         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 643         .tunnel_match = mlx5_flow_tunnel_match,
 644         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 645         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 646         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 647 };
 648
 649 /* Tunnel information. */
 650 struct mlx5_flow_tunnel_info {
 651         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 652         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 653 };
 654
 655 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 656         {
 657                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 658                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 659         },
 660         {
 661                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 662                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 663         },
 664         {
 665                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 666                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 667         },
 668         {
 669                 .tunnel = MLX5_FLOW_LAYER_GRE,
 670                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 671         },
 672         {
 673                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 674                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 675         },
 676         {
 677                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 678                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 679         },
 680         {
 681                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 682                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 683         },
 684         {
 685                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 686                 .ptype = RTE_PTYPE_TUNNEL_IP,
 687         },
 688         {
 689                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 690                 .ptype = RTE_PTYPE_TUNNEL_IP,
 691         },
 692         {
 693                 .tunnel = MLX5_FLOW_LAYER_GTP,
 694                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 695         },
 696 };
 697
 698 /* Key of thread specific flow workspace data. */
 699 static pthread_key_t key_workspace;
 700
 701 /* Thread specific flow workspace data once initialization data. */
 702 static pthread_once_t key_workspace_init;
 703
 704
 705 /**
 706  * Translate tag ID to register.
 707  *
 708  * @param[in] dev
 709  *   Pointer to the Ethernet device structure.
 710  * @param[in] feature
 711  *   The feature that request the register.
 712  * @param[in] id
 713  *   The request register ID.
 714  * @param[out] error
 715  *   Error description in case of any.
 716  *
 717  * @return
 718  *   The request register on success, a negative errno
 719  *   value otherwise and rte_errno is set.
 720  */
 721 int
 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 723                      enum mlx5_feature_name feature,
 724                      uint32_t id,
 725                      struct rte_flow_error *error)
 726 {
 727         struct mlx5_priv *priv = dev->data->dev_private;
 728         struct mlx5_dev_config *config = &priv->config;
 729         enum modify_reg start_reg;
 730         bool skip_mtr_reg = false;
 731
 732         switch (feature) {
 733         case MLX5_HAIRPIN_RX:
 734                 return REG_B;
 735         case MLX5_HAIRPIN_TX:
 736                 return REG_A;
 737         case MLX5_METADATA_RX:
 738                 switch (config->dv_xmeta_en) {
 739                 case MLX5_XMETA_MODE_LEGACY:
 740                         return REG_B;
 741                 case MLX5_XMETA_MODE_META16:
 742                         return REG_C_0;
 743                 case MLX5_XMETA_MODE_META32:
 744                         return REG_C_1;
 745                 }
 746                 break;
 747         case MLX5_METADATA_TX:
 748                 return REG_A;
 749         case MLX5_METADATA_FDB:
 750                 switch (config->dv_xmeta_en) {
 751                 case MLX5_XMETA_MODE_LEGACY:
 752                         return REG_NON;
 753                 case MLX5_XMETA_MODE_META16:
 754                         return REG_C_0;
 755                 case MLX5_XMETA_MODE_META32:
 756                         return REG_C_1;
 757                 }
 758                 break;
 759         case MLX5_FLOW_MARK:
 760                 switch (config->dv_xmeta_en) {
 761                 case MLX5_XMETA_MODE_LEGACY:
 762                         return REG_NON;
 763                 case MLX5_XMETA_MODE_META16:
 764                         return REG_C_1;
 765                 case MLX5_XMETA_MODE_META32:
 766                         return REG_C_0;
 767                 }
 768                 break;
 769         case MLX5_MTR_SFX:
 770                 /*
 771                  * If meter color and flow match share one register, flow match
 772                  * should use the meter color register for match.
 773                  */
 774                 if (priv->mtr_reg_share)
 775                         return priv->mtr_color_reg;
 776                 else
 777                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 778                                REG_C_3;
 779         case MLX5_MTR_COLOR:
 780                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 781                 return priv->mtr_color_reg;
 782         case MLX5_COPY_MARK:
 783                 /*
 784                  * Metadata COPY_MARK register using is in meter suffix sub
 785                  * flow while with meter. It's safe to share the same register.
 786                  */
 787                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 788         case MLX5_APP_TAG:
 789                 /*
 790                  * If meter is enable, it will engage the register for color
 791                  * match and flow match. If meter color match is not using the
 792                  * REG_C_2, need to skip the REG_C_x be used by meter color
 793                  * match.
 794                  * If meter is disable, free to use all available registers.
 795                  */
 796                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 797                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 798                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 799                 if (id > (REG_C_7 - start_reg))
 800                         return rte_flow_error_set(error, EINVAL,
 801                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 802                                                   NULL, "invalid tag id");
 803                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 804                         return rte_flow_error_set(error, ENOTSUP,
 805                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 806                                                   NULL, "unsupported tag id");
 807                 /*
 808                  * This case means meter is using the REG_C_x great than 2.
 809                  * Take care not to conflict with meter color REG_C_x.
 810                  * If the available index REG_C_y >= REG_C_x, skip the
 811                  * color register.
 812                  */
 813                 if (skip_mtr_reg && config->flow_mreg_c
 814                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 815                         if (id >= (REG_C_7 - start_reg))
 816                                 return rte_flow_error_set(error, EINVAL,
 817                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 818                                                         NULL, "invalid tag id");
 819                         if (config->flow_mreg_c
 820                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 821                                 return config->flow_mreg_c
 822                                                [id + 1 + start_reg - REG_C_0];
 823                         return rte_flow_error_set(error, ENOTSUP,
 824                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 825                                                   NULL, "unsupported tag id");
 826                 }
 827                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 828         }
 829         MLX5_ASSERT(false);
 830         return rte_flow_error_set(error, EINVAL,
 831                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 832                                   NULL, "invalid feature name");
 833 }
 834
 835 /**
 836  * Check extensive flow metadata register support.
 837  *
 838  * @param dev
 839  *   Pointer to rte_eth_dev structure.
 840  *
 841  * @return
 842  *   True if device supports extensive flow metadata register, otherwise false.
 843  */
 844 bool
 845 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 846 {
 847         struct mlx5_priv *priv = dev->data->dev_private;
 848         struct mlx5_dev_config *config = &priv->config;
 849
 850         /*
 851          * Having available reg_c can be regarded inclusively as supporting
 852          * extensive flow metadata register, which could mean,
 853          * - metadata register copy action by modify header.
 854          * - 16 modify header actions is supported.
 855          * - reg_c's are preserved across different domain (FDB and NIC) on
 856          *   packet loopback by flow lookup miss.
 857          */
 858         return config->flow_mreg_c[2] != REG_NON;
 859 }
 860
 861 /**
 862  * Verify the @p item specifications (spec, last, mask) are compatible with the
 863  * NIC capabilities.
 864  *
 865  * @param[in] item
 866  *   Item specification.
 867  * @param[in] mask
 868  *   @p item->mask or flow default bit-masks.
 869  * @param[in] nic_mask
 870  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 871  * @param[in] size
 872  *   Bit-masks size in bytes.
 873  * @param[in] range_accepted
 874  *   True if range of values is accepted for specific fields, false otherwise.
 875  * @param[out] error
 876  *   Pointer to error structure.
 877  *
 878  * @return
 879  *   0 on success, a negative errno value otherwise and rte_errno is set.
 880  */
 881 int
 882 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 883                           const uint8_t *mask,
 884                           const uint8_t *nic_mask,
 885                           unsigned int size,
 886                           bool range_accepted,
 887                           struct rte_flow_error *error)
 888 {
 889         unsigned int i;
 890
 891         MLX5_ASSERT(nic_mask);
 892         for (i = 0; i < size; ++i)
 893                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 894                         return rte_flow_error_set(error, ENOTSUP,
 895                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 896                                                   item,
 897                                                   "mask enables non supported"
 898                                                   " bits");
 899         if (!item->spec && (item->mask || item->last))
 900                 return rte_flow_error_set(error, EINVAL,
 901                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 902                                           "mask/last without a spec is not"
 903                                           " supported");
 904         if (item->spec && item->last && !range_accepted) {
 905                 uint8_t spec[size];
 906                 uint8_t last[size];
 907                 unsigned int i;
 908                 int ret;
 909
 910                 for (i = 0; i < size; ++i) {
 911                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 912                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 913                 }
 914                 ret = memcmp(spec, last, size);
 915                 if (ret != 0)
 916                         return rte_flow_error_set(error, EINVAL,
 917                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 918                                                   item,
 919                                                   "range is not valid");
 920         }
 921         return 0;
 922 }
 923
 924 /**
 925  * Adjust the hash fields according to the @p flow information.
 926  *
 927  * @param[in] dev_flow.
 928  *   Pointer to the mlx5_flow.
 929  * @param[in] tunnel
 930  *   1 when the hash field is for a tunnel item.
 931  * @param[in] layer_types
 932  *   ETH_RSS_* types.
 933  * @param[in] hash_fields
 934  *   Item hash fields.
 935  *
 936  * @return
 937  *   The hash fields that should be used.
 938  */
 939 uint64_t
 940 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 941                             int tunnel __rte_unused, uint64_t layer_types,
 942                             uint64_t hash_fields)
 943 {
 944 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 945         int rss_request_inner = rss_desc->level >= 2;
 946
 947         /* Check RSS hash level for tunnel. */
 948         if (tunnel && rss_request_inner)
 949                 hash_fields |= IBV_RX_HASH_INNER;
 950         else if (tunnel || rss_request_inner)
 951                 return 0;
 952 #endif
 953         /* Check if requested layer matches RSS hash fields. */
 954         if (!(rss_desc->types & layer_types))
 955                 return 0;
 956         return hash_fields;
 957 }
 958
 959 /**
 960  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 961  * if several tunnel rules are used on this queue, the tunnel ptype will be
 962  * cleared.
 963  *
 964  * @param rxq_ctrl
 965  *   Rx queue to update.
 966  */
 967 static void
 968 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 969 {
 970         unsigned int i;
 971         uint32_t tunnel_ptype = 0;
 972
 973         /* Look up for the ptype to use. */
 974         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 975                 if (!rxq_ctrl->flow_tunnels_n[i])
 976                         continue;
 977                 if (!tunnel_ptype) {
 978                         tunnel_ptype = tunnels_info[i].ptype;
 979                 } else {
 980                         tunnel_ptype = 0;
 981                         break;
 982                 }
 983         }
 984         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 985 }
 986
 987 /**
 988  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 989  * flow.
 990  *
 991  * @param[in] dev
 992  *   Pointer to the Ethernet device structure.
 993  * @param[in] dev_handle
 994  *   Pointer to device flow handle structure.
 995  */
 996 static void
 997 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
 998                        struct mlx5_flow_handle *dev_handle)
 999 {
1000         struct mlx5_priv *priv = dev->data->dev_private;
1001         const int mark = dev_handle->mark;
1002         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1003         struct mlx5_hrxq *hrxq;
1004         unsigned int i;
1005
1006         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1007                 return;
1008         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1009                               dev_handle->rix_hrxq);
1010         if (!hrxq)
1011                 return;
1012         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1013                 int idx = hrxq->ind_table->queues[i];
1014                 struct mlx5_rxq_ctrl *rxq_ctrl =
1015                         container_of((*priv->rxqs)[idx],
1016                                      struct mlx5_rxq_ctrl, rxq);
1017
1018                 /*
1019                  * To support metadata register copy on Tx loopback,
1020                  * this must be always enabled (metadata may arive
1021                  * from other port - not from local flows only.
1022                  */
1023                 if (priv->config.dv_flow_en &&
1024                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1025                     mlx5_flow_ext_mreg_supported(dev)) {
1026                         rxq_ctrl->rxq.mark = 1;
1027                         rxq_ctrl->flow_mark_n = 1;
1028                 } else if (mark) {
1029                         rxq_ctrl->rxq.mark = 1;
1030                         rxq_ctrl->flow_mark_n++;
1031                 }
1032                 if (tunnel) {
1033                         unsigned int j;
1034
1035                         /* Increase the counter matching the flow. */
1036                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1037                                 if ((tunnels_info[j].tunnel &
1038                                      dev_handle->layers) ==
1039                                     tunnels_info[j].tunnel) {
1040                                         rxq_ctrl->flow_tunnels_n[j]++;
1041                                         break;
1042                                 }
1043                         }
1044                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1045                 }
1046         }
1047 }
1048
1049 /**
1050  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1051  *
1052  * @param[in] dev
1053  *   Pointer to the Ethernet device structure.
1054  * @param[in] flow
1055  *   Pointer to flow structure.
1056  */
1057 static void
1058 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1059 {
1060         struct mlx5_priv *priv = dev->data->dev_private;
1061         uint32_t handle_idx;
1062         struct mlx5_flow_handle *dev_handle;
1063
1064         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1065                        handle_idx, dev_handle, next)
1066                 flow_drv_rxq_flags_set(dev, dev_handle);
1067 }
1068
1069 /**
1070  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1071  * device flow if no other flow uses it with the same kind of request.
1072  *
1073  * @param dev
1074  *   Pointer to Ethernet device.
1075  * @param[in] dev_handle
1076  *   Pointer to the device flow handle structure.
1077  */
1078 static void
1079 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1080                         struct mlx5_flow_handle *dev_handle)
1081 {
1082         struct mlx5_priv *priv = dev->data->dev_private;
1083         const int mark = dev_handle->mark;
1084         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1085         struct mlx5_hrxq *hrxq;
1086         unsigned int i;
1087
1088         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1089                 return;
1090         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1091                               dev_handle->rix_hrxq);
1092         if (!hrxq)
1093                 return;
1094         MLX5_ASSERT(dev->data->dev_started);
1095         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1096                 int idx = hrxq->ind_table->queues[i];
1097                 struct mlx5_rxq_ctrl *rxq_ctrl =
1098                         container_of((*priv->rxqs)[idx],
1099                                      struct mlx5_rxq_ctrl, rxq);
1100
1101                 if (priv->config.dv_flow_en &&
1102                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1103                     mlx5_flow_ext_mreg_supported(dev)) {
1104                         rxq_ctrl->rxq.mark = 1;
1105                         rxq_ctrl->flow_mark_n = 1;
1106                 } else if (mark) {
1107                         rxq_ctrl->flow_mark_n--;
1108                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1109                 }
1110                 if (tunnel) {
1111                         unsigned int j;
1112
1113                         /* Decrease the counter matching the flow. */
1114                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1115                                 if ((tunnels_info[j].tunnel &
1116                                      dev_handle->layers) ==
1117                                     tunnels_info[j].tunnel) {
1118                                         rxq_ctrl->flow_tunnels_n[j]--;
1119                                         break;
1120                                 }
1121                         }
1122                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1123                 }
1124         }
1125 }
1126
1127 /**
1128  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1129  * @p flow if no other flow uses it with the same kind of request.
1130  *
1131  * @param dev
1132  *   Pointer to Ethernet device.
1133  * @param[in] flow
1134  *   Pointer to the flow.
1135  */
1136 static void
1137 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1138 {
1139         struct mlx5_priv *priv = dev->data->dev_private;
1140         uint32_t handle_idx;
1141         struct mlx5_flow_handle *dev_handle;
1142
1143         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1144                        handle_idx, dev_handle, next)
1145                 flow_drv_rxq_flags_trim(dev, dev_handle);
1146 }
1147
1148 /**
1149  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1150  *
1151  * @param dev
1152  *   Pointer to Ethernet device.
1153  */
1154 static void
1155 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1156 {
1157         struct mlx5_priv *priv = dev->data->dev_private;
1158         unsigned int i;
1159
1160         for (i = 0; i != priv->rxqs_n; ++i) {
1161                 struct mlx5_rxq_ctrl *rxq_ctrl;
1162                 unsigned int j;
1163
1164                 if (!(*priv->rxqs)[i])
1165                         continue;
1166                 rxq_ctrl = container_of((*priv->rxqs)[i],
1167                                         struct mlx5_rxq_ctrl, rxq);
1168                 rxq_ctrl->flow_mark_n = 0;
1169                 rxq_ctrl->rxq.mark = 0;
1170                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1171                         rxq_ctrl->flow_tunnels_n[j] = 0;
1172                 rxq_ctrl->rxq.tunnel = 0;
1173         }
1174 }
1175
1176 /**
1177  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1178  *
1179  * @param[in] dev
1180  *   Pointer to the Ethernet device structure.
1181  */
1182 void
1183 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1184 {
1185         struct mlx5_priv *priv = dev->data->dev_private;
1186         struct mlx5_rxq_data *data;
1187         unsigned int i;
1188
1189         for (i = 0; i != priv->rxqs_n; ++i) {
1190                 if (!(*priv->rxqs)[i])
1191                         continue;
1192                 data = (*priv->rxqs)[i];
1193                 if (!rte_flow_dynf_metadata_avail()) {
1194                         data->dynf_meta = 0;
1195                         data->flow_meta_mask = 0;
1196                         data->flow_meta_offset = -1;
1197                 } else {
1198                         data->dynf_meta = 1;
1199                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1200                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1201                 }
1202         }
1203 }
1204
1205 /*
1206  * return a pointer to the desired action in the list of actions.
1207  *
1208  * @param[in] actions
1209  *   The list of actions to search the action in.
1210  * @param[in] action
1211  *   The action to find.
1212  *
1213  * @return
1214  *   Pointer to the action in the list, if found. NULL otherwise.
1215  */
1216 const struct rte_flow_action *
1217 mlx5_flow_find_action(const struct rte_flow_action *actions,
1218                       enum rte_flow_action_type action)
1219 {
1220         if (actions == NULL)
1221                 return NULL;
1222         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1223                 if (actions->type == action)
1224                         return actions;
1225         return NULL;
1226 }
1227
1228 /*
1229  * Validate the flag action.
1230  *
1231  * @param[in] action_flags
1232  *   Bit-fields that holds the actions detected until now.
1233  * @param[in] attr
1234  *   Attributes of flow that includes this action.
1235  * @param[out] error
1236  *   Pointer to error structure.
1237  *
1238  * @return
1239  *   0 on success, a negative errno value otherwise and rte_errno is set.
1240  */
1241 int
1242 mlx5_flow_validate_action_flag(uint64_t action_flags,
1243                                const struct rte_flow_attr *attr,
1244                                struct rte_flow_error *error)
1245 {
1246         if (action_flags & MLX5_FLOW_ACTION_MARK)
1247                 return rte_flow_error_set(error, EINVAL,
1248                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1249                                           "can't mark and flag in same flow");
1250         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1251                 return rte_flow_error_set(error, EINVAL,
1252                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1253                                           "can't have 2 flag"
1254                                           " actions in same flow");
1255         if (attr->egress)
1256                 return rte_flow_error_set(error, ENOTSUP,
1257                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1258                                           "flag action not supported for "
1259                                           "egress");
1260         return 0;
1261 }
1262
1263 /*
1264  * Validate the mark action.
1265  *
1266  * @param[in] action
1267  *   Pointer to the queue action.
1268  * @param[in] action_flags
1269  *   Bit-fields that holds the actions detected until now.
1270  * @param[in] attr
1271  *   Attributes of flow that includes this action.
1272  * @param[out] error
1273  *   Pointer to error structure.
1274  *
1275  * @return
1276  *   0 on success, a negative errno value otherwise and rte_errno is set.
1277  */
1278 int
1279 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1280                                uint64_t action_flags,
1281                                const struct rte_flow_attr *attr,
1282                                struct rte_flow_error *error)
1283 {
1284         const struct rte_flow_action_mark *mark = action->conf;
1285
1286         if (!mark)
1287                 return rte_flow_error_set(error, EINVAL,
1288                                           RTE_FLOW_ERROR_TYPE_ACTION,
1289                                           action,
1290                                           "configuration cannot be null");
1291         if (mark->id >= MLX5_FLOW_MARK_MAX)
1292                 return rte_flow_error_set(error, EINVAL,
1293                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1294                                           &mark->id,
1295                                           "mark id must in 0 <= id < "
1296                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1297         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1298                 return rte_flow_error_set(error, EINVAL,
1299                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1300                                           "can't flag and mark in same flow");
1301         if (action_flags & MLX5_FLOW_ACTION_MARK)
1302                 return rte_flow_error_set(error, EINVAL,
1303                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1304                                           "can't have 2 mark actions in same"
1305                                           " flow");
1306         if (attr->egress)
1307                 return rte_flow_error_set(error, ENOTSUP,
1308                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1309                                           "mark action not supported for "
1310                                           "egress");
1311         return 0;
1312 }
1313
1314 /*
1315  * Validate the drop action.
1316  *
1317  * @param[in] action_flags
1318  *   Bit-fields that holds the actions detected until now.
1319  * @param[in] attr
1320  *   Attributes of flow that includes this action.
1321  * @param[out] error
1322  *   Pointer to error structure.
1323  *
1324  * @return
1325  *   0 on success, a negative errno value otherwise and rte_errno is set.
1326  */
1327 int
1328 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1329                                const struct rte_flow_attr *attr,
1330                                struct rte_flow_error *error)
1331 {
1332         if (attr->egress)
1333                 return rte_flow_error_set(error, ENOTSUP,
1334                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1335                                           "drop action not supported for "
1336                                           "egress");
1337         return 0;
1338 }
1339
1340 /*
1341  * Validate the queue action.
1342  *
1343  * @param[in] action
1344  *   Pointer to the queue action.
1345  * @param[in] action_flags
1346  *   Bit-fields that holds the actions detected until now.
1347  * @param[in] dev
1348  *   Pointer to the Ethernet device structure.
1349  * @param[in] attr
1350  *   Attributes of flow that includes this action.
1351  * @param[out] error
1352  *   Pointer to error structure.
1353  *
1354  * @return
1355  *   0 on success, a negative errno value otherwise and rte_errno is set.
1356  */
1357 int
1358 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1359                                 uint64_t action_flags,
1360                                 struct rte_eth_dev *dev,
1361                                 const struct rte_flow_attr *attr,
1362                                 struct rte_flow_error *error)
1363 {
1364         struct mlx5_priv *priv = dev->data->dev_private;
1365         const struct rte_flow_action_queue *queue = action->conf;
1366
1367         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1368                 return rte_flow_error_set(error, EINVAL,
1369                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1370                                           "can't have 2 fate actions in"
1371                                           " same flow");
1372         if (!priv->rxqs_n)
1373                 return rte_flow_error_set(error, EINVAL,
1374                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1375                                           NULL, "No Rx queues configured");
1376         if (queue->index >= priv->rxqs_n)
1377                 return rte_flow_error_set(error, EINVAL,
1378                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1379                                           &queue->index,
1380                                           "queue index out of range");
1381         if (!(*priv->rxqs)[queue->index])
1382                 return rte_flow_error_set(error, EINVAL,
1383                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1384                                           &queue->index,
1385                                           "queue is not configured");
1386         if (attr->egress)
1387                 return rte_flow_error_set(error, ENOTSUP,
1388                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1389                                           "queue action not supported for "
1390                                           "egress");
1391         return 0;
1392 }
1393
1394 /*
1395  * Validate the rss action.
1396  *
1397  * @param[in] dev
1398  *   Pointer to the Ethernet device structure.
1399  * @param[in] action
1400  *   Pointer to the queue action.
1401  * @param[out] error
1402  *   Pointer to error structure.
1403  *
1404  * @return
1405  *   0 on success, a negative errno value otherwise and rte_errno is set.
1406  */
1407 int
1408 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1409                          const struct rte_flow_action *action,
1410                          struct rte_flow_error *error)
1411 {
1412         struct mlx5_priv *priv = dev->data->dev_private;
1413         const struct rte_flow_action_rss *rss = action->conf;
1414         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1415         unsigned int i;
1416
1417         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1418             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1419                 return rte_flow_error_set(error, ENOTSUP,
1420                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1421                                           &rss->func,
1422                                           "RSS hash function not supported");
1423 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1424         if (rss->level > 2)
1425 #else
1426         if (rss->level > 1)
1427 #endif
1428                 return rte_flow_error_set(error, ENOTSUP,
1429                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1430                                           &rss->level,
1431                                           "tunnel RSS is not supported");
1432         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1433         if (rss->key_len == 0 && rss->key != NULL)
1434                 return rte_flow_error_set(error, ENOTSUP,
1435                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1436                                           &rss->key_len,
1437                                           "RSS hash key length 0");
1438         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1439                 return rte_flow_error_set(error, ENOTSUP,
1440                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1441                                           &rss->key_len,
1442                                           "RSS hash key too small");
1443         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1444                 return rte_flow_error_set(error, ENOTSUP,
1445                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1446                                           &rss->key_len,
1447                                           "RSS hash key too large");
1448         if (rss->queue_num > priv->config.ind_table_max_size)
1449                 return rte_flow_error_set(error, ENOTSUP,
1450                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1451                                           &rss->queue_num,
1452                                           "number of queues too large");
1453         if (rss->types & MLX5_RSS_HF_MASK)
1454                 return rte_flow_error_set(error, ENOTSUP,
1455                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1456                                           &rss->types,
1457                                           "some RSS protocols are not"
1458                                           " supported");
1459         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1460             !(rss->types & ETH_RSS_IP))
1461                 return rte_flow_error_set(error, EINVAL,
1462                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1463                                           "L3 partial RSS requested but L3 RSS"
1464                                           " type not specified");
1465         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1466             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1469                                           "L4 partial RSS requested but L4 RSS"
1470                                           " type not specified");
1471         if (!priv->rxqs_n)
1472                 return rte_flow_error_set(error, EINVAL,
1473                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1474                                           NULL, "No Rx queues configured");
1475         if (!rss->queue_num)
1476                 return rte_flow_error_set(error, EINVAL,
1477                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1478                                           NULL, "No queues configured");
1479         for (i = 0; i != rss->queue_num; ++i) {
1480                 struct mlx5_rxq_ctrl *rxq_ctrl;
1481
1482                 if (rss->queue[i] >= priv->rxqs_n)
1483                         return rte_flow_error_set
1484                                 (error, EINVAL,
1485                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1486                                  &rss->queue[i], "queue index out of range");
1487                 if (!(*priv->rxqs)[rss->queue[i]])
1488                         return rte_flow_error_set
1489                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1490                                  &rss->queue[i], "queue is not configured");
1491                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1492                                         struct mlx5_rxq_ctrl, rxq);
1493                 if (i == 0)
1494                         rxq_type = rxq_ctrl->type;
1495                 if (rxq_type != rxq_ctrl->type)
1496                         return rte_flow_error_set
1497                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1498                                  &rss->queue[i],
1499                                  "combining hairpin and regular RSS queues is not supported");
1500         }
1501         return 0;
1502 }
1503
1504 /*
1505  * Validate the rss action.
1506  *
1507  * @param[in] action
1508  *   Pointer to the queue action.
1509  * @param[in] action_flags
1510  *   Bit-fields that holds the actions detected until now.
1511  * @param[in] dev
1512  *   Pointer to the Ethernet device structure.
1513  * @param[in] attr
1514  *   Attributes of flow that includes this action.
1515  * @param[in] item_flags
1516  *   Items that were detected.
1517  * @param[out] error
1518  *   Pointer to error structure.
1519  *
1520  * @return
1521  *   0 on success, a negative errno value otherwise and rte_errno is set.
1522  */
1523 int
1524 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1525                               uint64_t action_flags,
1526                               struct rte_eth_dev *dev,
1527                               const struct rte_flow_attr *attr,
1528                               uint64_t item_flags,
1529                               struct rte_flow_error *error)
1530 {
1531         const struct rte_flow_action_rss *rss = action->conf;
1532         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1533         int ret;
1534
1535         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1536                 return rte_flow_error_set(error, EINVAL,
1537                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1538                                           "can't have 2 fate actions"
1539                                           " in same flow");
1540         ret = mlx5_validate_action_rss(dev, action, error);
1541         if (ret)
1542                 return ret;
1543         if (attr->egress)
1544                 return rte_flow_error_set(error, ENOTSUP,
1545                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1546                                           "rss action not supported for "
1547                                           "egress");
1548         if (rss->level > 1 && !tunnel)
1549                 return rte_flow_error_set(error, EINVAL,
1550                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1551                                           "inner RSS is not supported for "
1552                                           "non-tunnel flows");
1553         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1554             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1555                 return rte_flow_error_set(error, EINVAL,
1556                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1557                                           "RSS on eCPRI is not supported now");
1558         }
1559         return 0;
1560 }
1561
1562 /*
1563  * Validate the default miss action.
1564  *
1565  * @param[in] action_flags
1566  *   Bit-fields that holds the actions detected until now.
1567  * @param[out] error
1568  *   Pointer to error structure.
1569  *
1570  * @return
1571  *   0 on success, a negative errno value otherwise and rte_errno is set.
1572  */
1573 int
1574 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1575                                 const struct rte_flow_attr *attr,
1576                                 struct rte_flow_error *error)
1577 {
1578         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1579                 return rte_flow_error_set(error, EINVAL,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1581                                           "can't have 2 fate actions in"
1582                                           " same flow");
1583         if (attr->egress)
1584                 return rte_flow_error_set(error, ENOTSUP,
1585                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1586                                           "default miss action not supported "
1587                                           "for egress");
1588         if (attr->group)
1589                 return rte_flow_error_set(error, ENOTSUP,
1590                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1591                                           "only group 0 is supported");
1592         if (attr->transfer)
1593                 return rte_flow_error_set(error, ENOTSUP,
1594                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1595                                           NULL, "transfer is not supported");
1596         return 0;
1597 }
1598
1599 /*
1600  * Validate the count action.
1601  *
1602  * @param[in] dev
1603  *   Pointer to the Ethernet device structure.
1604  * @param[in] attr
1605  *   Attributes of flow that includes this action.
1606  * @param[out] error
1607  *   Pointer to error structure.
1608  *
1609  * @return
1610  *   0 on success, a negative errno value otherwise and rte_errno is set.
1611  */
1612 int
1613 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1614                                 const struct rte_flow_attr *attr,
1615                                 struct rte_flow_error *error)
1616 {
1617         if (attr->egress)
1618                 return rte_flow_error_set(error, ENOTSUP,
1619                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1620                                           "count action not supported for "
1621                                           "egress");
1622         return 0;
1623 }
1624
1625 /**
1626  * Verify the @p attributes will be correctly understood by the NIC and store
1627  * them in the @p flow if everything is correct.
1628  *
1629  * @param[in] dev
1630  *   Pointer to the Ethernet device structure.
1631  * @param[in] attributes
1632  *   Pointer to flow attributes
1633  * @param[out] error
1634  *   Pointer to error structure.
1635  *
1636  * @return
1637  *   0 on success, a negative errno value otherwise and rte_errno is set.
1638  */
1639 int
1640 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1641                               const struct rte_flow_attr *attributes,
1642                               struct rte_flow_error *error)
1643 {
1644         struct mlx5_priv *priv = dev->data->dev_private;
1645         uint32_t priority_max = priv->config.flow_prio - 1;
1646
1647         if (attributes->group)
1648                 return rte_flow_error_set(error, ENOTSUP,
1649                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1650                                           NULL, "groups is not supported");
1651         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1652             attributes->priority >= priority_max)
1653                 return rte_flow_error_set(error, ENOTSUP,
1654                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1655                                           NULL, "priority out of range");
1656         if (attributes->egress)
1657                 return rte_flow_error_set(error, ENOTSUP,
1658                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1659                                           "egress is not supported");
1660         if (attributes->transfer && !priv->config.dv_esw_en)
1661                 return rte_flow_error_set(error, ENOTSUP,
1662                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1663                                           NULL, "transfer is not supported");
1664         if (!attributes->ingress)
1665                 return rte_flow_error_set(error, EINVAL,
1666                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1667                                           NULL,
1668                                           "ingress attribute is mandatory");
1669         return 0;
1670 }
1671
1672 /**
1673  * Validate ICMP6 item.
1674  *
1675  * @param[in] item
1676  *   Item specification.
1677  * @param[in] item_flags
1678  *   Bit-fields that holds the items detected until now.
1679  * @param[in] ext_vlan_sup
1680  *   Whether extended VLAN features are supported or not.
1681  * @param[out] error
1682  *   Pointer to error structure.
1683  *
1684  * @return
1685  *   0 on success, a negative errno value otherwise and rte_errno is set.
1686  */
1687 int
1688 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1689                                uint64_t item_flags,
1690                                uint8_t target_protocol,
1691                                struct rte_flow_error *error)
1692 {
1693         const struct rte_flow_item_icmp6 *mask = item->mask;
1694         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1695         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1696                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1697         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1698                                       MLX5_FLOW_LAYER_OUTER_L4;
1699         int ret;
1700
1701         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1702                 return rte_flow_error_set(error, EINVAL,
1703                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1704                                           "protocol filtering not compatible"
1705                                           " with ICMP6 layer");
1706         if (!(item_flags & l3m))
1707                 return rte_flow_error_set(error, EINVAL,
1708                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1709                                           "IPv6 is mandatory to filter on"
1710                                           " ICMP6");
1711         if (item_flags & l4m)
1712                 return rte_flow_error_set(error, EINVAL,
1713                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1714                                           "multiple L4 layers not supported");
1715         if (!mask)
1716                 mask = &rte_flow_item_icmp6_mask;
1717         ret = mlx5_flow_item_acceptable
1718                 (item, (const uint8_t *)mask,
1719                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1720                  sizeof(struct rte_flow_item_icmp6),
1721                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1722         if (ret < 0)
1723                 return ret;
1724         return 0;
1725 }
1726
1727 /**
1728  * Validate ICMP item.
1729  *
1730  * @param[in] item
1731  *   Item specification.
1732  * @param[in] item_flags
1733  *   Bit-fields that holds the items detected until now.
1734  * @param[out] error
1735  *   Pointer to error structure.
1736  *
1737  * @return
1738  *   0 on success, a negative errno value otherwise and rte_errno is set.
1739  */
1740 int
1741 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1742                              uint64_t item_flags,
1743                              uint8_t target_protocol,
1744                              struct rte_flow_error *error)
1745 {
1746         const struct rte_flow_item_icmp *mask = item->mask;
1747         const struct rte_flow_item_icmp nic_mask = {
1748                 .hdr.icmp_type = 0xff,
1749                 .hdr.icmp_code = 0xff,
1750                 .hdr.icmp_ident = RTE_BE16(0xffff),
1751                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1752         };
1753         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1754         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1755                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1756         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1757                                       MLX5_FLOW_LAYER_OUTER_L4;
1758         int ret;
1759
1760         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1761                 return rte_flow_error_set(error, EINVAL,
1762                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1763                                           "protocol filtering not compatible"
1764                                           " with ICMP layer");
1765         if (!(item_flags & l3m))
1766                 return rte_flow_error_set(error, EINVAL,
1767                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1768                                           "IPv4 is mandatory to filter"
1769                                           " on ICMP");
1770         if (item_flags & l4m)
1771                 return rte_flow_error_set(error, EINVAL,
1772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1773                                           "multiple L4 layers not supported");
1774         if (!mask)
1775                 mask = &nic_mask;
1776         ret = mlx5_flow_item_acceptable
1777                 (item, (const uint8_t *)mask,
1778                  (const uint8_t *)&nic_mask,
1779                  sizeof(struct rte_flow_item_icmp),
1780                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1781         if (ret < 0)
1782                 return ret;
1783         return 0;
1784 }
1785
1786 /**
1787  * Validate Ethernet item.
1788  *
1789  * @param[in] item
1790  *   Item specification.
1791  * @param[in] item_flags
1792  *   Bit-fields that holds the items detected until now.
1793  * @param[out] error
1794  *   Pointer to error structure.
1795  *
1796  * @return
1797  *   0 on success, a negative errno value otherwise and rte_errno is set.
1798  */
1799 int
1800 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1801                             uint64_t item_flags, bool ext_vlan_sup,
1802                             struct rte_flow_error *error)
1803 {
1804         const struct rte_flow_item_eth *mask = item->mask;
1805         const struct rte_flow_item_eth nic_mask = {
1806                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1807                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1808                 .type = RTE_BE16(0xffff),
1809                 .has_vlan = ext_vlan_sup ? 1 : 0,
1810         };
1811         int ret;
1812         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1813         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1814                                        MLX5_FLOW_LAYER_OUTER_L2;
1815
1816         if (item_flags & ethm)
1817                 return rte_flow_error_set(error, ENOTSUP,
1818                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1819                                           "multiple L2 layers not supported");
1820         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1821             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1822                 return rte_flow_error_set(error, EINVAL,
1823                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1824                                           "L2 layer should not follow "
1825                                           "L3 layers");
1826         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1827             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1828                 return rte_flow_error_set(error, EINVAL,
1829                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1830                                           "L2 layer should not follow VLAN");
1831         if (!mask)
1832                 mask = &rte_flow_item_eth_mask;
1833         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1834                                         (const uint8_t *)&nic_mask,
1835                                         sizeof(struct rte_flow_item_eth),
1836                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1837         return ret;
1838 }
1839
1840 /**
1841  * Validate VLAN item.
1842  *
1843  * @param[in] item
1844  *   Item specification.
1845  * @param[in] item_flags
1846  *   Bit-fields that holds the items detected until now.
1847  * @param[in] dev
1848  *   Ethernet device flow is being created on.
1849  * @param[out] error
1850  *   Pointer to error structure.
1851  *
1852  * @return
1853  *   0 on success, a negative errno value otherwise and rte_errno is set.
1854  */
1855 int
1856 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1857                              uint64_t item_flags,
1858                              struct rte_eth_dev *dev,
1859                              struct rte_flow_error *error)
1860 {
1861         const struct rte_flow_item_vlan *spec = item->spec;
1862         const struct rte_flow_item_vlan *mask = item->mask;
1863         const struct rte_flow_item_vlan nic_mask = {
1864                 .tci = RTE_BE16(UINT16_MAX),
1865                 .inner_type = RTE_BE16(UINT16_MAX),
1866         };
1867         uint16_t vlan_tag = 0;
1868         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1869         int ret;
1870         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1871                                         MLX5_FLOW_LAYER_INNER_L4) :
1872                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1873                                         MLX5_FLOW_LAYER_OUTER_L4);
1874         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1875                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1876
1877         if (item_flags & vlanm)
1878                 return rte_flow_error_set(error, EINVAL,
1879                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1880                                           "multiple VLAN layers not supported");
1881         else if ((item_flags & l34m) != 0)
1882                 return rte_flow_error_set(error, EINVAL,
1883                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1884                                           "VLAN cannot follow L3/L4 layer");
1885         if (!mask)
1886                 mask = &rte_flow_item_vlan_mask;
1887         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1888                                         (const uint8_t *)&nic_mask,
1889                                         sizeof(struct rte_flow_item_vlan),
1890                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1891         if (ret)
1892                 return ret;
1893         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1894                 struct mlx5_priv *priv = dev->data->dev_private;
1895
1896                 if (priv->vmwa_context) {
1897                         /*
1898                          * Non-NULL context means we have a virtual machine
1899                          * and SR-IOV enabled, we have to create VLAN interface
1900                          * to make hypervisor to setup E-Switch vport
1901                          * context correctly. We avoid creating the multiple
1902                          * VLAN interfaces, so we cannot support VLAN tag mask.
1903                          */
1904                         return rte_flow_error_set(error, EINVAL,
1905                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1906                                                   item,
1907                                                   "VLAN tag mask is not"
1908                                                   " supported in virtual"
1909                                                   " environment");
1910                 }
1911         }
1912         if (spec) {
1913                 vlan_tag = spec->tci;
1914                 vlan_tag &= mask->tci;
1915         }
1916         /*
1917          * From verbs perspective an empty VLAN is equivalent
1918          * to a packet without VLAN layer.
1919          */
1920         if (!vlan_tag)
1921                 return rte_flow_error_set(error, EINVAL,
1922                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1923                                           item->spec,
1924                                           "VLAN cannot be empty");
1925         return 0;
1926 }
1927
1928 /**
1929  * Validate IPV4 item.
1930  *
1931  * @param[in] item
1932  *   Item specification.
1933  * @param[in] item_flags
1934  *   Bit-fields that holds the items detected until now.
1935  * @param[in] last_item
1936  *   Previous validated item in the pattern items.
1937  * @param[in] ether_type
1938  *   Type in the ethernet layer header (including dot1q).
1939  * @param[in] acc_mask
1940  *   Acceptable mask, if NULL default internal default mask
1941  *   will be used to check whether item fields are supported.
1942  * @param[in] range_accepted
1943  *   True if range of values is accepted for specific fields, false otherwise.
1944  * @param[out] error
1945  *   Pointer to error structure.
1946  *
1947  * @return
1948  *   0 on success, a negative errno value otherwise and rte_errno is set.
1949  */
1950 int
1951 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1952                              uint64_t item_flags,
1953                              uint64_t last_item,
1954                              uint16_t ether_type,
1955                              const struct rte_flow_item_ipv4 *acc_mask,
1956                              bool range_accepted,
1957                              struct rte_flow_error *error)
1958 {
1959         const struct rte_flow_item_ipv4 *mask = item->mask;
1960         const struct rte_flow_item_ipv4 *spec = item->spec;
1961         const struct rte_flow_item_ipv4 nic_mask = {
1962                 .hdr = {
1963                         .src_addr = RTE_BE32(0xffffffff),
1964                         .dst_addr = RTE_BE32(0xffffffff),
1965                         .type_of_service = 0xff,
1966                         .next_proto_id = 0xff,
1967                 },
1968         };
1969         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1970         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1971                                       MLX5_FLOW_LAYER_OUTER_L3;
1972         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1973                                       MLX5_FLOW_LAYER_OUTER_L4;
1974         int ret;
1975         uint8_t next_proto = 0xFF;
1976         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1977                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1978                                   MLX5_FLOW_LAYER_INNER_VLAN);
1979
1980         if ((last_item & l2_vlan) && ether_type &&
1981             ether_type != RTE_ETHER_TYPE_IPV4)
1982                 return rte_flow_error_set(error, EINVAL,
1983                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1984                                           "IPv4 cannot follow L2/VLAN layer "
1985                                           "which ether type is not IPv4");
1986         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1987                 if (mask && spec)
1988                         next_proto = mask->hdr.next_proto_id &
1989                                      spec->hdr.next_proto_id;
1990                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1991                         return rte_flow_error_set(error, EINVAL,
1992                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1993                                                   item,
1994                                                   "multiple tunnel "
1995                                                   "not supported");
1996         }
1997         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1998                 return rte_flow_error_set(error, EINVAL,
1999                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2000                                           "wrong tunnel type - IPv6 specified "
2001                                           "but IPv4 item provided");
2002         if (item_flags & l3m)
2003                 return rte_flow_error_set(error, ENOTSUP,
2004                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2005                                           "multiple L3 layers not supported");
2006         else if (item_flags & l4m)
2007                 return rte_flow_error_set(error, EINVAL,
2008                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2009                                           "L3 cannot follow an L4 layer.");
2010         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2011                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2012                 return rte_flow_error_set(error, EINVAL,
2013                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2014                                           "L3 cannot follow an NVGRE layer.");
2015         if (!mask)
2016                 mask = &rte_flow_item_ipv4_mask;
2017         else if (mask->hdr.next_proto_id != 0 &&
2018                  mask->hdr.next_proto_id != 0xff)
2019                 return rte_flow_error_set(error, EINVAL,
2020                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2021                                           "partial mask is not supported"
2022                                           " for protocol");
2023         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2024                                         acc_mask ? (const uint8_t *)acc_mask
2025                                                  : (const uint8_t *)&nic_mask,
2026                                         sizeof(struct rte_flow_item_ipv4),
2027                                         range_accepted, error);
2028         if (ret < 0)
2029                 return ret;
2030         return 0;
2031 }
2032
2033 /**
2034  * Validate IPV6 item.
2035  *
2036  * @param[in] item
2037  *   Item specification.
2038  * @param[in] item_flags
2039  *   Bit-fields that holds the items detected until now.
2040  * @param[in] last_item
2041  *   Previous validated item in the pattern items.
2042  * @param[in] ether_type
2043  *   Type in the ethernet layer header (including dot1q).
2044  * @param[in] acc_mask
2045  *   Acceptable mask, if NULL default internal default mask
2046  *   will be used to check whether item fields are supported.
2047  * @param[out] error
2048  *   Pointer to error structure.
2049  *
2050  * @return
2051  *   0 on success, a negative errno value otherwise and rte_errno is set.
2052  */
2053 int
2054 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2055                              uint64_t item_flags,
2056                              uint64_t last_item,
2057                              uint16_t ether_type,
2058                              const struct rte_flow_item_ipv6 *acc_mask,
2059                              struct rte_flow_error *error)
2060 {
2061         const struct rte_flow_item_ipv6 *mask = item->mask;
2062         const struct rte_flow_item_ipv6 *spec = item->spec;
2063         const struct rte_flow_item_ipv6 nic_mask = {
2064                 .hdr = {
2065                         .src_addr =
2066                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2067                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2068                         .dst_addr =
2069                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2070                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2071                         .vtc_flow = RTE_BE32(0xffffffff),
2072                         .proto = 0xff,
2073                 },
2074         };
2075         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2076         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2077                                       MLX5_FLOW_LAYER_OUTER_L3;
2078         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2079                                       MLX5_FLOW_LAYER_OUTER_L4;
2080         int ret;
2081         uint8_t next_proto = 0xFF;
2082         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2083                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2084                                   MLX5_FLOW_LAYER_INNER_VLAN);
2085
2086         if ((last_item & l2_vlan) && ether_type &&
2087             ether_type != RTE_ETHER_TYPE_IPV6)
2088                 return rte_flow_error_set(error, EINVAL,
2089                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2090                                           "IPv6 cannot follow L2/VLAN layer "
2091                                           "which ether type is not IPv6");
2092         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2093                 next_proto = spec->hdr.proto;
2094         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2095                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2096                         return rte_flow_error_set(error, EINVAL,
2097                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2098                                                   item,
2099                                                   "multiple tunnel "
2100                                                   "not supported");
2101         }
2102         if (next_proto == IPPROTO_HOPOPTS  ||
2103             next_proto == IPPROTO_ROUTING  ||
2104             next_proto == IPPROTO_FRAGMENT ||
2105             next_proto == IPPROTO_ESP      ||
2106             next_proto == IPPROTO_AH       ||
2107             next_proto == IPPROTO_DSTOPTS)
2108                 return rte_flow_error_set(error, EINVAL,
2109                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2110                                           "IPv6 proto (next header) should "
2111                                           "not be set as extension header");
2112         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2113                 return rte_flow_error_set(error, EINVAL,
2114                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2115                                           "wrong tunnel type - IPv4 specified "
2116                                           "but IPv6 item provided");
2117         if (item_flags & l3m)
2118                 return rte_flow_error_set(error, ENOTSUP,
2119                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2120                                           "multiple L3 layers not supported");
2121         else if (item_flags & l4m)
2122                 return rte_flow_error_set(error, EINVAL,
2123                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2124                                           "L3 cannot follow an L4 layer.");
2125         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2126                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2127                 return rte_flow_error_set(error, EINVAL,
2128                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2129                                           "L3 cannot follow an NVGRE layer.");
2130         if (!mask)
2131                 mask = &rte_flow_item_ipv6_mask;
2132         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2133                                         acc_mask ? (const uint8_t *)acc_mask
2134                                                  : (const uint8_t *)&nic_mask,
2135                                         sizeof(struct rte_flow_item_ipv6),
2136                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2137         if (ret < 0)
2138                 return ret;
2139         return 0;
2140 }
2141
2142 /**
2143  * Validate UDP item.
2144  *
2145  * @param[in] item
2146  *   Item specification.
2147  * @param[in] item_flags
2148  *   Bit-fields that holds the items detected until now.
2149  * @param[in] target_protocol
2150  *   The next protocol in the previous item.
2151  * @param[in] flow_mask
2152  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2153  * @param[out] error
2154  *   Pointer to error structure.
2155  *
2156  * @return
2157  *   0 on success, a negative errno value otherwise and rte_errno is set.
2158  */
2159 int
2160 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2161                             uint64_t item_flags,
2162                             uint8_t target_protocol,
2163                             struct rte_flow_error *error)
2164 {
2165         const struct rte_flow_item_udp *mask = item->mask;
2166         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2167         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2168                                       MLX5_FLOW_LAYER_OUTER_L3;
2169         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2170                                       MLX5_FLOW_LAYER_OUTER_L4;
2171         int ret;
2172
2173         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2174                 return rte_flow_error_set(error, EINVAL,
2175                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2176                                           "protocol filtering not compatible"
2177                                           " with UDP layer");
2178         if (!(item_flags & l3m))
2179                 return rte_flow_error_set(error, EINVAL,
2180                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2181                                           "L3 is mandatory to filter on L4");
2182         if (item_flags & l4m)
2183                 return rte_flow_error_set(error, EINVAL,
2184                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2185                                           "multiple L4 layers not supported");
2186         if (!mask)
2187                 mask = &rte_flow_item_udp_mask;
2188         ret = mlx5_flow_item_acceptable
2189                 (item, (const uint8_t *)mask,
2190                  (const uint8_t *)&rte_flow_item_udp_mask,
2191                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2192                  error);
2193         if (ret < 0)
2194                 return ret;
2195         return 0;
2196 }
2197
2198 /**
2199  * Validate TCP item.
2200  *
2201  * @param[in] item
2202  *   Item specification.
2203  * @param[in] item_flags
2204  *   Bit-fields that holds the items detected until now.
2205  * @param[in] target_protocol
2206  *   The next protocol in the previous item.
2207  * @param[out] error
2208  *   Pointer to error structure.
2209  *
2210  * @return
2211  *   0 on success, a negative errno value otherwise and rte_errno is set.
2212  */
2213 int
2214 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2215                             uint64_t item_flags,
2216                             uint8_t target_protocol,
2217                             const struct rte_flow_item_tcp *flow_mask,
2218                             struct rte_flow_error *error)
2219 {
2220         const struct rte_flow_item_tcp *mask = item->mask;
2221         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2222         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2223                                       MLX5_FLOW_LAYER_OUTER_L3;
2224         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2225                                       MLX5_FLOW_LAYER_OUTER_L4;
2226         int ret;
2227
2228         MLX5_ASSERT(flow_mask);
2229         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2230                 return rte_flow_error_set(error, EINVAL,
2231                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2232                                           "protocol filtering not compatible"
2233                                           " with TCP layer");
2234         if (!(item_flags & l3m))
2235                 return rte_flow_error_set(error, EINVAL,
2236                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2237                                           "L3 is mandatory to filter on L4");
2238         if (item_flags & l4m)
2239                 return rte_flow_error_set(error, EINVAL,
2240                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2241                                           "multiple L4 layers not supported");
2242         if (!mask)
2243                 mask = &rte_flow_item_tcp_mask;
2244         ret = mlx5_flow_item_acceptable
2245                 (item, (const uint8_t *)mask,
2246                  (const uint8_t *)flow_mask,
2247                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2248                  error);
2249         if (ret < 0)
2250                 return ret;
2251         return 0;
2252 }
2253
2254 /**
2255  * Validate VXLAN item.
2256  *
2257  * @param[in] item
2258  *   Item specification.
2259  * @param[in] item_flags
2260  *   Bit-fields that holds the items detected until now.
2261  * @param[in] target_protocol
2262  *   The next protocol in the previous item.
2263  * @param[out] error
2264  *   Pointer to error structure.
2265  *
2266  * @return
2267  *   0 on success, a negative errno value otherwise and rte_errno is set.
2268  */
2269 int
2270 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2271                               uint64_t item_flags,
2272                               struct rte_flow_error *error)
2273 {
2274         const struct rte_flow_item_vxlan *spec = item->spec;
2275         const struct rte_flow_item_vxlan *mask = item->mask;
2276         int ret;
2277         union vni {
2278                 uint32_t vlan_id;
2279                 uint8_t vni[4];
2280         } id = { .vlan_id = 0, };
2281
2282
2283         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2284                 return rte_flow_error_set(error, ENOTSUP,
2285                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2286                                           "multiple tunnel layers not"
2287                                           " supported");
2288         /*
2289          * Verify only UDPv4 is present as defined in
2290          * https://tools.ietf.org/html/rfc7348
2291          */
2292         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2293                 return rte_flow_error_set(error, EINVAL,
2294                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2295                                           "no outer UDP layer found");
2296         if (!mask)
2297                 mask = &rte_flow_item_vxlan_mask;
2298         ret = mlx5_flow_item_acceptable
2299                 (item, (const uint8_t *)mask,
2300                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2301                  sizeof(struct rte_flow_item_vxlan),
2302                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2303         if (ret < 0)
2304                 return ret;
2305         if (spec) {
2306                 memcpy(&id.vni[1], spec->vni, 3);
2307                 memcpy(&id.vni[1], mask->vni, 3);
2308         }
2309         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2310                 return rte_flow_error_set(error, ENOTSUP,
2311                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2312                                           "VXLAN tunnel must be fully defined");
2313         return 0;
2314 }
2315
2316 /**
2317  * Validate VXLAN_GPE item.
2318  *
2319  * @param[in] item
2320  *   Item specification.
2321  * @param[in] item_flags
2322  *   Bit-fields that holds the items detected until now.
2323  * @param[in] priv
2324  *   Pointer to the private data structure.
2325  * @param[in] target_protocol
2326  *   The next protocol in the previous item.
2327  * @param[out] error
2328  *   Pointer to error structure.
2329  *
2330  * @return
2331  *   0 on success, a negative errno value otherwise and rte_errno is set.
2332  */
2333 int
2334 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2335                                   uint64_t item_flags,
2336                                   struct rte_eth_dev *dev,
2337                                   struct rte_flow_error *error)
2338 {
2339         struct mlx5_priv *priv = dev->data->dev_private;
2340         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2341         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2342         int ret;
2343         union vni {
2344                 uint32_t vlan_id;
2345                 uint8_t vni[4];
2346         } id = { .vlan_id = 0, };
2347
2348         if (!priv->config.l3_vxlan_en)
2349                 return rte_flow_error_set(error, ENOTSUP,
2350                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2351                                           "L3 VXLAN is not enabled by device"
2352                                           " parameter and/or not configured in"
2353                                           " firmware");
2354         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2355                 return rte_flow_error_set(error, ENOTSUP,
2356                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2357                                           "multiple tunnel layers not"
2358                                           " supported");
2359         /*
2360          * Verify only UDPv4 is present as defined in
2361          * https://tools.ietf.org/html/rfc7348
2362          */
2363         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2364                 return rte_flow_error_set(error, EINVAL,
2365                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2366                                           "no outer UDP layer found");
2367         if (!mask)
2368                 mask = &rte_flow_item_vxlan_gpe_mask;
2369         ret = mlx5_flow_item_acceptable
2370                 (item, (const uint8_t *)mask,
2371                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2372                  sizeof(struct rte_flow_item_vxlan_gpe),
2373                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2374         if (ret < 0)
2375                 return ret;
2376         if (spec) {
2377                 if (spec->protocol)
2378                         return rte_flow_error_set(error, ENOTSUP,
2379                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2380                                                   item,
2381                                                   "VxLAN-GPE protocol"
2382                                                   " not supported");
2383                 memcpy(&id.vni[1], spec->vni, 3);
2384                 memcpy(&id.vni[1], mask->vni, 3);
2385         }
2386         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2387                 return rte_flow_error_set(error, ENOTSUP,
2388                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2389                                           "VXLAN-GPE tunnel must be fully"
2390                                           " defined");
2391         return 0;
2392 }
2393 /**
2394  * Validate GRE Key item.
2395  *
2396  * @param[in] item
2397  *   Item specification.
2398  * @param[in] item_flags
2399  *   Bit flags to mark detected items.
2400  * @param[in] gre_item
2401  *   Pointer to gre_item
2402  * @param[out] error
2403  *   Pointer to error structure.
2404  *
2405  * @return
2406  *   0 on success, a negative errno value otherwise and rte_errno is set.
2407  */
2408 int
2409 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2410                                 uint64_t item_flags,
2411                                 const struct rte_flow_item *gre_item,
2412                                 struct rte_flow_error *error)
2413 {
2414         const rte_be32_t *mask = item->mask;
2415         int ret = 0;
2416         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2417         const struct rte_flow_item_gre *gre_spec;
2418         const struct rte_flow_item_gre *gre_mask;
2419
2420         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2421                 return rte_flow_error_set(error, ENOTSUP,
2422                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2423                                           "Multiple GRE key not support");
2424         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2425                 return rte_flow_error_set(error, ENOTSUP,
2426                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2427                                           "No preceding GRE header");
2428         if (item_flags & MLX5_FLOW_LAYER_INNER)
2429                 return rte_flow_error_set(error, ENOTSUP,
2430                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2431                                           "GRE key following a wrong item");
2432         gre_mask = gre_item->mask;
2433         if (!gre_mask)
2434                 gre_mask = &rte_flow_item_gre_mask;
2435         gre_spec = gre_item->spec;
2436         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2437                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2438                 return rte_flow_error_set(error, EINVAL,
2439                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2440                                           "Key bit must be on");
2441
2442         if (!mask)
2443                 mask = &gre_key_default_mask;
2444         ret = mlx5_flow_item_acceptable
2445                 (item, (const uint8_t *)mask,
2446                  (const uint8_t *)&gre_key_default_mask,
2447                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2448         return ret;
2449 }
2450
2451 /**
2452  * Validate GRE item.
2453  *
2454  * @param[in] item
2455  *   Item specification.
2456  * @param[in] item_flags
2457  *   Bit flags to mark detected items.
2458  * @param[in] target_protocol
2459  *   The next protocol in the previous item.
2460  * @param[out] error
2461  *   Pointer to error structure.
2462  *
2463  * @return
2464  *   0 on success, a negative errno value otherwise and rte_errno is set.
2465  */
2466 int
2467 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2468                             uint64_t item_flags,
2469                             uint8_t target_protocol,
2470                             struct rte_flow_error *error)
2471 {
2472         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2473         const struct rte_flow_item_gre *mask = item->mask;
2474         int ret;
2475         const struct rte_flow_item_gre nic_mask = {
2476                 .c_rsvd0_ver = RTE_BE16(0xB000),
2477                 .protocol = RTE_BE16(UINT16_MAX),
2478         };
2479
2480         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2481                 return rte_flow_error_set(error, EINVAL,
2482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2483                                           "protocol filtering not compatible"
2484                                           " with this GRE layer");
2485         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2486                 return rte_flow_error_set(error, ENOTSUP,
2487                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2488                                           "multiple tunnel layers not"
2489                                           " supported");
2490         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2491                 return rte_flow_error_set(error, ENOTSUP,
2492                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2493                                           "L3 Layer is missing");
2494         if (!mask)
2495                 mask = &rte_flow_item_gre_mask;
2496         ret = mlx5_flow_item_acceptable
2497                 (item, (const uint8_t *)mask,
2498                  (const uint8_t *)&nic_mask,
2499                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2500                  error);
2501         if (ret < 0)
2502                 return ret;
2503 #ifndef HAVE_MLX5DV_DR
2504 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2505         if (spec && (spec->protocol & mask->protocol))
2506                 return rte_flow_error_set(error, ENOTSUP,
2507                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2508                                           "without MPLS support the"
2509                                           " specification cannot be used for"
2510                                           " filtering");
2511 #endif
2512 #endif
2513         return 0;
2514 }
2515
2516 /**
2517  * Validate Geneve item.
2518  *
2519  * @param[in] item
2520  *   Item specification.
2521  * @param[in] itemFlags
2522  *   Bit-fields that holds the items detected until now.
2523  * @param[in] enPriv
2524  *   Pointer to the private data structure.
2525  * @param[out] error
2526  *   Pointer to error structure.
2527  *
2528  * @return
2529  *   0 on success, a negative errno value otherwise and rte_errno is set.
2530  */
2531
2532 int
2533 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2534                                uint64_t item_flags,
2535                                struct rte_eth_dev *dev,
2536                                struct rte_flow_error *error)
2537 {
2538         struct mlx5_priv *priv = dev->data->dev_private;
2539         const struct rte_flow_item_geneve *spec = item->spec;
2540         const struct rte_flow_item_geneve *mask = item->mask;
2541         int ret;
2542         uint16_t gbhdr;
2543         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2544                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2545         const struct rte_flow_item_geneve nic_mask = {
2546                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2547                 .vni = "\xff\xff\xff",
2548                 .protocol = RTE_BE16(UINT16_MAX),
2549         };
2550
2551         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2552                 return rte_flow_error_set(error, ENOTSUP,
2553                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2554                                           "L3 Geneve is not enabled by device"
2555                                           " parameter and/or not configured in"
2556                                           " firmware");
2557         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2558                 return rte_flow_error_set(error, ENOTSUP,
2559                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2560                                           "multiple tunnel layers not"
2561                                           " supported");
2562         /*
2563          * Verify only UDPv4 is present as defined in
2564          * https://tools.ietf.org/html/rfc7348
2565          */
2566         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2567                 return rte_flow_error_set(error, EINVAL,
2568                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2569                                           "no outer UDP layer found");
2570         if (!mask)
2571                 mask = &rte_flow_item_geneve_mask;
2572         ret = mlx5_flow_item_acceptable
2573                                   (item, (const uint8_t *)mask,
2574                                    (const uint8_t *)&nic_mask,
2575                                    sizeof(struct rte_flow_item_geneve),
2576                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2577         if (ret)
2578                 return ret;
2579         if (spec) {
2580                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2581                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2582                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2583                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2584                         return rte_flow_error_set(error, ENOTSUP,
2585                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2586                                                   item,
2587                                                   "Geneve protocol unsupported"
2588                                                   " fields are being used");
2589                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2590                         return rte_flow_error_set
2591                                         (error, ENOTSUP,
2592                                          RTE_FLOW_ERROR_TYPE_ITEM,
2593                                          item,
2594                                          "Unsupported Geneve options length");
2595         }
2596         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2597                 return rte_flow_error_set
2598                                     (error, ENOTSUP,
2599                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2600                                      "Geneve tunnel must be fully defined");
2601         return 0;
2602 }
2603
2604 /**
2605  * Validate MPLS item.
2606  *
2607  * @param[in] dev
2608  *   Pointer to the rte_eth_dev structure.
2609  * @param[in] item
2610  *   Item specification.
2611  * @param[in] item_flags
2612  *   Bit-fields that holds the items detected until now.
2613  * @param[in] prev_layer
2614  *   The protocol layer indicated in previous item.
2615  * @param[out] error
2616  *   Pointer to error structure.
2617  *
2618  * @return
2619  *   0 on success, a negative errno value otherwise and rte_errno is set.
2620  */
2621 int
2622 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2623                              const struct rte_flow_item *item __rte_unused,
2624                              uint64_t item_flags __rte_unused,
2625                              uint64_t prev_layer __rte_unused,
2626                              struct rte_flow_error *error)
2627 {
2628 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2629         const struct rte_flow_item_mpls *mask = item->mask;
2630         struct mlx5_priv *priv = dev->data->dev_private;
2631         int ret;
2632
2633         if (!priv->config.mpls_en)
2634                 return rte_flow_error_set(error, ENOTSUP,
2635                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2636                                           "MPLS not supported or"
2637                                           " disabled in firmware"
2638                                           " configuration.");
2639         /* MPLS over IP, UDP, GRE is allowed */
2640         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2641                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2642                             MLX5_FLOW_LAYER_GRE |
2643                             MLX5_FLOW_LAYER_GRE_KEY)))
2644                 return rte_flow_error_set(error, EINVAL,
2645                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2646                                           "protocol filtering not compatible"
2647                                           " with MPLS layer");
2648         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2649         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2650             !(item_flags & MLX5_FLOW_LAYER_GRE))
2651                 return rte_flow_error_set(error, ENOTSUP,
2652                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2653                                           "multiple tunnel layers not"
2654                                           " supported");
2655         if (!mask)
2656                 mask = &rte_flow_item_mpls_mask;
2657         ret = mlx5_flow_item_acceptable
2658                 (item, (const uint8_t *)mask,
2659                  (const uint8_t *)&rte_flow_item_mpls_mask,
2660                  sizeof(struct rte_flow_item_mpls),
2661                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2662         if (ret < 0)
2663                 return ret;
2664         return 0;
2665 #else
2666         return rte_flow_error_set(error, ENOTSUP,
2667                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2668                                   "MPLS is not supported by Verbs, please"
2669                                   " update.");
2670 #endif
2671 }
2672
2673 /**
2674  * Validate NVGRE item.
2675  *
2676  * @param[in] item
2677  *   Item specification.
2678  * @param[in] item_flags
2679  *   Bit flags to mark detected items.
2680  * @param[in] target_protocol
2681  *   The next protocol in the previous item.
2682  * @param[out] error
2683  *   Pointer to error structure.
2684  *
2685  * @return
2686  *   0 on success, a negative errno value otherwise and rte_errno is set.
2687  */
2688 int
2689 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2690                               uint64_t item_flags,
2691                               uint8_t target_protocol,
2692                               struct rte_flow_error *error)
2693 {
2694         const struct rte_flow_item_nvgre *mask = item->mask;
2695         int ret;
2696
2697         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2698                 return rte_flow_error_set(error, EINVAL,
2699                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2700                                           "protocol filtering not compatible"
2701                                           " with this GRE layer");
2702         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2703                 return rte_flow_error_set(error, ENOTSUP,
2704                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2705                                           "multiple tunnel layers not"
2706                                           " supported");
2707         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2708                 return rte_flow_error_set(error, ENOTSUP,
2709                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2710                                           "L3 Layer is missing");
2711         if (!mask)
2712                 mask = &rte_flow_item_nvgre_mask;
2713         ret = mlx5_flow_item_acceptable
2714                 (item, (const uint8_t *)mask,
2715                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2716                  sizeof(struct rte_flow_item_nvgre),
2717                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2718         if (ret < 0)
2719                 return ret;
2720         return 0;
2721 }
2722
2723 /**
2724  * Validate eCPRI item.
2725  *
2726  * @param[in] item
2727  *   Item specification.
2728  * @param[in] item_flags
2729  *   Bit-fields that holds the items detected until now.
2730  * @param[in] last_item
2731  *   Previous validated item in the pattern items.
2732  * @param[in] ether_type
2733  *   Type in the ethernet layer header (including dot1q).
2734  * @param[in] acc_mask
2735  *   Acceptable mask, if NULL default internal default mask
2736  *   will be used to check whether item fields are supported.
2737  * @param[out] error
2738  *   Pointer to error structure.
2739  *
2740  * @return
2741  *   0 on success, a negative errno value otherwise and rte_errno is set.
2742  */
2743 int
2744 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2745                               uint64_t item_flags,
2746                               uint64_t last_item,
2747                               uint16_t ether_type,
2748                               const struct rte_flow_item_ecpri *acc_mask,
2749                               struct rte_flow_error *error)
2750 {
2751         const struct rte_flow_item_ecpri *mask = item->mask;
2752         const struct rte_flow_item_ecpri nic_mask = {
2753                 .hdr = {
2754                         .common = {
2755                                 .u32 =
2756                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2757                                         .type = 0xFF,
2758                                         }).u32),
2759                         },
2760                         .dummy[0] = 0xFFFFFFFF,
2761                 },
2762         };
2763         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2764                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2765         struct rte_flow_item_ecpri mask_lo;
2766
2767         if (!(last_item & outer_l2_vlan) &&
2768             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
2769                 return rte_flow_error_set(error, EINVAL,
2770                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2771                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
2772         if ((last_item & outer_l2_vlan) && ether_type &&
2773             ether_type != RTE_ETHER_TYPE_ECPRI)
2774                 return rte_flow_error_set(error, EINVAL,
2775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2776                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
2777         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2778                 return rte_flow_error_set(error, EINVAL,
2779                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2780                                           "eCPRI with tunnel is not supported right now");
2781         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2782                 return rte_flow_error_set(error, ENOTSUP,
2783                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2784                                           "multiple L3 layers not supported");
2785         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2786                 return rte_flow_error_set(error, EINVAL,
2787                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2788                                           "eCPRI cannot coexist with a TCP layer");
2789         /* In specification, eCPRI could be over UDP layer. */
2790         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2791                 return rte_flow_error_set(error, EINVAL,
2792                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2793                                           "eCPRI over UDP layer is not yet supported right now");
2794         /* Mask for type field in common header could be zero. */
2795         if (!mask)
2796                 mask = &rte_flow_item_ecpri_mask;
2797         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2798         /* Input mask is in big-endian format. */
2799         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2800                 return rte_flow_error_set(error, EINVAL,
2801                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2802                                           "partial mask is not supported for protocol");
2803         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2804                 return rte_flow_error_set(error, EINVAL,
2805                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2806                                           "message header mask must be after a type mask");
2807         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2808                                          acc_mask ? (const uint8_t *)acc_mask
2809                                                   : (const uint8_t *)&nic_mask,
2810                                          sizeof(struct rte_flow_item_ecpri),
2811                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2812 }
2813
2814 /**
2815  * Release resource related QUEUE/RSS action split.
2816  *
2817  * @param dev
2818  *   Pointer to Ethernet device.
2819  * @param flow
2820  *   Flow to release id's from.
2821  */
2822 static void
2823 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2824                              struct rte_flow *flow)
2825 {
2826         struct mlx5_priv *priv = dev->data->dev_private;
2827         uint32_t handle_idx;
2828         struct mlx5_flow_handle *dev_handle;
2829
2830         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2831                        handle_idx, dev_handle, next)
2832                 if (dev_handle->split_flow_id)
2833                         mlx5_ipool_free(priv->sh->ipool
2834                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
2835                                         dev_handle->split_flow_id);
2836 }
2837
2838 static int
2839 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2840                    const struct rte_flow_attr *attr __rte_unused,
2841                    const struct rte_flow_item items[] __rte_unused,
2842                    const struct rte_flow_action actions[] __rte_unused,
2843                    bool external __rte_unused,
2844                    int hairpin __rte_unused,
2845                    struct rte_flow_error *error)
2846 {
2847         return rte_flow_error_set(error, ENOTSUP,
2848                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2849 }
2850
2851 static struct mlx5_flow *
2852 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2853                   const struct rte_flow_attr *attr __rte_unused,
2854                   const struct rte_flow_item items[] __rte_unused,
2855                   const struct rte_flow_action actions[] __rte_unused,
2856                   struct rte_flow_error *error)
2857 {
2858         rte_flow_error_set(error, ENOTSUP,
2859                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2860         return NULL;
2861 }
2862
2863 static int
2864 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2865                     struct mlx5_flow *dev_flow __rte_unused,
2866                     const struct rte_flow_attr *attr __rte_unused,
2867                     const struct rte_flow_item items[] __rte_unused,
2868                     const struct rte_flow_action actions[] __rte_unused,
2869                     struct rte_flow_error *error)
2870 {
2871         return rte_flow_error_set(error, ENOTSUP,
2872                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2873 }
2874
2875 static int
2876 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2877                 struct rte_flow *flow __rte_unused,
2878                 struct rte_flow_error *error)
2879 {
2880         return rte_flow_error_set(error, ENOTSUP,
2881                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2882 }
2883
2884 static void
2885 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2886                  struct rte_flow *flow __rte_unused)
2887 {
2888 }
2889
2890 static void
2891 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2892                   struct rte_flow *flow __rte_unused)
2893 {
2894 }
2895
2896 static int
2897 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2898                 struct rte_flow *flow __rte_unused,
2899                 const struct rte_flow_action *actions __rte_unused,
2900                 void *data __rte_unused,
2901                 struct rte_flow_error *error)
2902 {
2903         return rte_flow_error_set(error, ENOTSUP,
2904                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2905 }
2906
2907 static int
2908 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
2909                       uint32_t domains __rte_unused,
2910                       uint32_t flags __rte_unused)
2911 {
2912         return 0;
2913 }
2914
2915 /* Void driver to protect from null pointer reference. */
2916 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2917         .validate = flow_null_validate,
2918         .prepare = flow_null_prepare,
2919         .translate = flow_null_translate,
2920         .apply = flow_null_apply,
2921         .remove = flow_null_remove,
2922         .destroy = flow_null_destroy,
2923         .query = flow_null_query,
2924         .sync_domain = flow_null_sync_domain,
2925 };
2926
2927 /**
2928  * Select flow driver type according to flow attributes and device
2929  * configuration.
2930  *
2931  * @param[in] dev
2932  *   Pointer to the dev structure.
2933  * @param[in] attr
2934  *   Pointer to the flow attributes.
2935  *
2936  * @return
2937  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2938  */
2939 static enum mlx5_flow_drv_type
2940 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2941 {
2942         struct mlx5_priv *priv = dev->data->dev_private;
2943         /* The OS can determine first a specific flow type (DV, VERBS) */
2944         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2945
2946         if (type != MLX5_FLOW_TYPE_MAX)
2947                 return type;
2948         /* If no OS specific type - continue with DV/VERBS selection */
2949         if (attr->transfer && priv->config.dv_esw_en)
2950                 type = MLX5_FLOW_TYPE_DV;
2951         if (!attr->transfer)
2952                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2953                                                  MLX5_FLOW_TYPE_VERBS;
2954         return type;
2955 }
2956
2957 #define flow_get_drv_ops(type) flow_drv_ops[type]
2958
2959 /**
2960  * Flow driver validation API. This abstracts calling driver specific functions.
2961  * The type of flow driver is determined according to flow attributes.
2962  *
2963  * @param[in] dev
2964  *   Pointer to the dev structure.
2965  * @param[in] attr
2966  *   Pointer to the flow attributes.
2967  * @param[in] items
2968  *   Pointer to the list of items.
2969  * @param[in] actions
2970  *   Pointer to the list of actions.
2971  * @param[in] external
2972  *   This flow rule is created by request external to PMD.
2973  * @param[in] hairpin
2974  *   Number of hairpin TX actions, 0 means classic flow.
2975  * @param[out] error
2976  *   Pointer to the error structure.
2977  *
2978  * @return
2979  *   0 on success, a negative errno value otherwise and rte_errno is set.
2980  */
2981 static inline int
2982 flow_drv_validate(struct rte_eth_dev *dev,
2983                   const struct rte_flow_attr *attr,
2984                   const struct rte_flow_item items[],
2985                   const struct rte_flow_action actions[],
2986                   bool external, int hairpin, struct rte_flow_error *error)
2987 {
2988         const struct mlx5_flow_driver_ops *fops;
2989         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2990
2991         fops = flow_get_drv_ops(type);
2992         return fops->validate(dev, attr, items, actions, external,
2993                               hairpin, error);
2994 }
2995
2996 /**
2997  * Flow driver preparation API. This abstracts calling driver specific
2998  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2999  * calculates the size of memory required for device flow, allocates the memory,
3000  * initializes the device flow and returns the pointer.
3001  *
3002  * @note
3003  *   This function initializes device flow structure such as dv or verbs in
3004  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3005  *   rest. For example, adding returning device flow to flow->dev_flow list and
3006  *   setting backward reference to the flow should be done out of this function.
3007  *   layers field is not filled either.
3008  *
3009  * @param[in] dev
3010  *   Pointer to the dev structure.
3011  * @param[in] attr
3012  *   Pointer to the flow attributes.
3013  * @param[in] items
3014  *   Pointer to the list of items.
3015  * @param[in] actions
3016  *   Pointer to the list of actions.
3017  * @param[in] flow_idx
3018  *   This memory pool index to the flow.
3019  * @param[out] error
3020  *   Pointer to the error structure.
3021  *
3022  * @return
3023  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3024  */
3025 static inline struct mlx5_flow *
3026 flow_drv_prepare(struct rte_eth_dev *dev,
3027                  const struct rte_flow *flow,
3028                  const struct rte_flow_attr *attr,
3029                  const struct rte_flow_item items[],
3030                  const struct rte_flow_action actions[],
3031                  uint32_t flow_idx,
3032                  struct rte_flow_error *error)
3033 {
3034         const struct mlx5_flow_driver_ops *fops;
3035         enum mlx5_flow_drv_type type = flow->drv_type;
3036         struct mlx5_flow *mlx5_flow = NULL;
3037
3038         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3039         fops = flow_get_drv_ops(type);
3040         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3041         if (mlx5_flow)
3042                 mlx5_flow->flow_idx = flow_idx;
3043         return mlx5_flow;
3044 }
3045
3046 /**
3047  * Flow driver translation API. This abstracts calling driver specific
3048  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3049  * translates a generic flow into a driver flow. flow_drv_prepare() must
3050  * precede.
3051  *
3052  * @note
3053  *   dev_flow->layers could be filled as a result of parsing during translation
3054  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3055  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3056  *   flow->actions could be overwritten even though all the expanded dev_flows
3057  *   have the same actions.
3058  *
3059  * @param[in] dev
3060  *   Pointer to the rte dev structure.
3061  * @param[in, out] dev_flow
3062  *   Pointer to the mlx5 flow.
3063  * @param[in] attr
3064  *   Pointer to the flow attributes.
3065  * @param[in] items
3066  *   Pointer to the list of items.
3067  * @param[in] actions
3068  *   Pointer to the list of actions.
3069  * @param[out] error
3070  *   Pointer to the error structure.
3071  *
3072  * @return
3073  *   0 on success, a negative errno value otherwise and rte_errno is set.
3074  */
3075 static inline int
3076 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3077                    const struct rte_flow_attr *attr,
3078                    const struct rte_flow_item items[],
3079                    const struct rte_flow_action actions[],
3080                    struct rte_flow_error *error)
3081 {
3082         const struct mlx5_flow_driver_ops *fops;
3083         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3084
3085         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3086         fops = flow_get_drv_ops(type);
3087         return fops->translate(dev, dev_flow, attr, items, actions, error);
3088 }
3089
3090 /**
3091  * Flow driver apply API. This abstracts calling driver specific functions.
3092  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3093  * translated driver flows on to device. flow_drv_translate() must precede.
3094  *
3095  * @param[in] dev
3096  *   Pointer to Ethernet device structure.
3097  * @param[in, out] flow
3098  *   Pointer to flow structure.
3099  * @param[out] error
3100  *   Pointer to error structure.
3101  *
3102  * @return
3103  *   0 on success, a negative errno value otherwise and rte_errno is set.
3104  */
3105 static inline int
3106 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3107                struct rte_flow_error *error)
3108 {
3109         const struct mlx5_flow_driver_ops *fops;
3110         enum mlx5_flow_drv_type type = flow->drv_type;
3111
3112         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3113         fops = flow_get_drv_ops(type);
3114         return fops->apply(dev, flow, error);
3115 }
3116
3117 /**
3118  * Flow driver destroy API. This abstracts calling driver specific functions.
3119  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3120  * on device and releases resources of the flow.
3121  *
3122  * @param[in] dev
3123  *   Pointer to Ethernet device.
3124  * @param[in, out] flow
3125  *   Pointer to flow structure.
3126  */
3127 static inline void
3128 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3129 {
3130         const struct mlx5_flow_driver_ops *fops;
3131         enum mlx5_flow_drv_type type = flow->drv_type;
3132
3133         flow_mreg_split_qrss_release(dev, flow);
3134         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3135         fops = flow_get_drv_ops(type);
3136         fops->destroy(dev, flow);
3137 }
3138
3139 /**
3140  * Get RSS action from the action list.
3141  *
3142  * @param[in] actions
3143  *   Pointer to the list of actions.
3144  *
3145  * @return
3146  *   Pointer to the RSS action if exist, else return NULL.
3147  */
3148 static const struct rte_flow_action_rss*
3149 flow_get_rss_action(const struct rte_flow_action actions[])
3150 {
3151         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3152                 switch (actions->type) {
3153                 case RTE_FLOW_ACTION_TYPE_RSS:
3154                         return (const struct rte_flow_action_rss *)
3155                                actions->conf;
3156                 default:
3157                         break;
3158                 }
3159         }
3160         return NULL;
3161 }
3162
3163 /**
3164  * Get ASO age action by index.
3165  *
3166  * @param[in] dev
3167  *   Pointer to the Ethernet device structure.
3168  * @param[in] age_idx
3169  *   Index to the ASO age action.
3170  *
3171  * @return
3172  *   The specified ASO age action.
3173  */
3174 struct mlx5_aso_age_action*
3175 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3176 {
3177         uint16_t pool_idx = age_idx & UINT16_MAX;
3178         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3179         struct mlx5_priv *priv = dev->data->dev_private;
3180         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3181         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3182
3183         return &pool->actions[offset - 1];
3184 }
3185
3186 /* maps shared action to translated non shared in some actions array */
3187 struct mlx5_translated_shared_action {
3188         struct rte_flow_shared_action *action; /**< Shared action */
3189         int index; /**< Index in related array of rte_flow_action */
3190 };
3191
3192 /**
3193  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3194  * non shared action if translation possible.
3195  * This functionality used to run same execution path for both shared & non
3196  * shared actions on flow create. All necessary preparations for shared
3197  * action handling should be preformed on *shared* actions list returned
3198  * from this call.
3199  *
3200  * @param[in] dev
3201  *   Pointer to Ethernet device.
3202  * @param[in] actions
3203  *   List of actions to translate.
3204  * @param[out] shared
3205  *   List to store translated shared actions.
3206  * @param[in, out] shared_n
3207  *   Size of *shared* array. On return should be updated with number of shared
3208  *   actions retrieved from the *actions* list.
3209  * @param[out] translated_actions
3210  *   List of actions where all shared actions were translated to non shared
3211  *   if possible. NULL if no translation took place.
3212  * @param[out] error
3213  *   Pointer to the error structure.
3214  *
3215  * @return
3216  *   0 on success, a negative errno value otherwise and rte_errno is set.
3217  */
3218 static int
3219 flow_shared_actions_translate(struct rte_eth_dev *dev,
3220                               const struct rte_flow_action actions[],
3221                               struct mlx5_translated_shared_action *shared,
3222                               int *shared_n,
3223                               struct rte_flow_action **translated_actions,
3224                               struct rte_flow_error *error)
3225 {
3226         struct mlx5_priv *priv = dev->data->dev_private;
3227         struct rte_flow_action *translated = NULL;
3228         size_t actions_size;
3229         int n;
3230         int copied_n = 0;
3231         struct mlx5_translated_shared_action *shared_end = NULL;
3232
3233         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3234                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3235                         continue;
3236                 if (copied_n == *shared_n) {
3237                         return rte_flow_error_set
3238                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3239                                  NULL, "too many shared actions");
3240                 }
3241                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3242                            sizeof(actions[n].conf));
3243                 shared[copied_n].index = n;
3244                 copied_n++;
3245         }
3246         n++;
3247         *shared_n = copied_n;
3248         if (!copied_n)
3249                 return 0;
3250         actions_size = sizeof(struct rte_flow_action) * n;
3251         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3252         if (!translated) {
3253                 rte_errno = ENOMEM;
3254                 return -ENOMEM;
3255         }
3256         memcpy(translated, actions, actions_size);
3257         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3258                 struct mlx5_shared_action_rss *shared_rss;
3259                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3260                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3261                 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET)
3262                                                                            - 1);
3263
3264                 switch (type) {
3265                 case MLX5_SHARED_ACTION_TYPE_RSS:
3266                         shared_rss = mlx5_ipool_get
3267                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3268                         translated[shared->index].type =
3269                                 RTE_FLOW_ACTION_TYPE_RSS;
3270                         translated[shared->index].conf =
3271                                 &shared_rss->origin;
3272                         break;
3273                 case MLX5_SHARED_ACTION_TYPE_AGE:
3274                         if (priv->sh->flow_hit_aso_en) {
3275                                 translated[shared->index].type =
3276                                         (enum rte_flow_action_type)
3277                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3278                                 translated[shared->index].conf =
3279                                                          (void *)(uintptr_t)idx;
3280                                 break;
3281                         }
3282                         /* Fall-through */
3283                 default:
3284                         mlx5_free(translated);
3285                         return rte_flow_error_set
3286                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3287                                  NULL, "invalid shared action type");
3288                 }
3289         }
3290         *translated_actions = translated;
3291         return 0;
3292 }
3293
3294 /**
3295  * Get Shared RSS action from the action list.
3296  *
3297  * @param[in] dev
3298  *   Pointer to Ethernet device.
3299  * @param[in] shared
3300  *   Pointer to the list of actions.
3301  * @param[in] shared_n
3302  *   Actions list length.
3303  *
3304  * @return
3305  *   The MLX5 RSS action ID if exists, otherwise return 0.
3306  */
3307 static uint32_t
3308 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3309                            struct mlx5_translated_shared_action *shared,
3310                            int shared_n)
3311 {
3312         struct mlx5_translated_shared_action *shared_end;
3313         struct mlx5_priv *priv = dev->data->dev_private;
3314         struct mlx5_shared_action_rss *shared_rss;
3315
3316
3317         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3318                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3319                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3320                 uint32_t idx = act_idx &
3321                                    ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1);
3322                 switch (type) {
3323                 case MLX5_SHARED_ACTION_TYPE_RSS:
3324                         shared_rss = mlx5_ipool_get
3325                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3326                                                                            idx);
3327                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3328                                            __ATOMIC_RELAXED);
3329                         return idx;
3330                 default:
3331                         break;
3332                 }
3333         }
3334         return 0;
3335 }
3336
3337 static unsigned int
3338 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3339 {
3340         const struct rte_flow_item *item;
3341         unsigned int has_vlan = 0;
3342
3343         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3344                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3345                         has_vlan = 1;
3346                         break;
3347                 }
3348         }
3349         if (has_vlan)
3350                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3351                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3352         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3353                                MLX5_EXPANSION_ROOT_OUTER;
3354 }
3355
3356 /**
3357  *  Get layer flags from the prefix flow.
3358  *
3359  *  Some flows may be split to several subflows, the prefix subflow gets the
3360  *  match items and the suffix sub flow gets the actions.
3361  *  Some actions need the user defined match item flags to get the detail for
3362  *  the action.
3363  *  This function helps the suffix flow to get the item layer flags from prefix
3364  *  subflow.
3365  *
3366  * @param[in] dev_flow
3367  *   Pointer the created preifx subflow.
3368  *
3369  * @return
3370  *   The layers get from prefix subflow.
3371  */
3372 static inline uint64_t
3373 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3374 {
3375         uint64_t layers = 0;
3376
3377         /*
3378          * Layers bits could be localization, but usually the compiler will
3379          * help to do the optimization work for source code.
3380          * If no decap actions, use the layers directly.
3381          */
3382         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3383                 return dev_flow->handle->layers;
3384         /* Convert L3 layers with decap action. */
3385         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3386                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3387         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3388                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3389         /* Convert L4 layers with decap action.  */
3390         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3391                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3392         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3393                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3394         return layers;
3395 }
3396
3397 /**
3398  * Get metadata split action information.
3399  *
3400  * @param[in] actions
3401  *   Pointer to the list of actions.
3402  * @param[out] qrss
3403  *   Pointer to the return pointer.
3404  * @param[out] qrss_type
3405  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3406  *   if no QUEUE/RSS is found.
3407  * @param[out] encap_idx
3408  *   Pointer to the index of the encap action if exists, otherwise the last
3409  *   action index.
3410  *
3411  * @return
3412  *   Total number of actions.
3413  */
3414 static int
3415 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3416                                        const struct rte_flow_action **qrss,
3417                                        int *encap_idx)
3418 {
3419         const struct rte_flow_action_raw_encap *raw_encap;
3420         int actions_n = 0;
3421         int raw_decap_idx = -1;
3422
3423         *encap_idx = -1;
3424         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3425                 switch (actions->type) {
3426                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3427                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3428                         *encap_idx = actions_n;
3429                         break;
3430                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3431                         raw_decap_idx = actions_n;
3432                         break;
3433                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3434                         raw_encap = actions->conf;
3435                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3436                                 *encap_idx = raw_decap_idx != -1 ?
3437                                                       raw_decap_idx : actions_n;
3438                         break;
3439                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3440                 case RTE_FLOW_ACTION_TYPE_RSS:
3441                         *qrss = actions;
3442                         break;
3443                 default:
3444                         break;
3445                 }
3446                 actions_n++;
3447         }
3448         if (*encap_idx == -1)
3449                 *encap_idx = actions_n;
3450         /* Count RTE_FLOW_ACTION_TYPE_END. */
3451         return actions_n + 1;
3452 }
3453
3454 /**
3455  * Check meter action from the action list.
3456  *
3457  * @param[in] actions
3458  *   Pointer to the list of actions.
3459  * @param[out] mtr
3460  *   Pointer to the meter exist flag.
3461  *
3462  * @return
3463  *   Total number of actions.
3464  */
3465 static int
3466 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3467 {
3468         int actions_n = 0;
3469
3470         MLX5_ASSERT(mtr);
3471         *mtr = 0;
3472         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3473                 switch (actions->type) {
3474                 case RTE_FLOW_ACTION_TYPE_METER:
3475                         *mtr = 1;
3476                         break;
3477                 default:
3478                         break;
3479                 }
3480                 actions_n++;
3481         }
3482         /* Count RTE_FLOW_ACTION_TYPE_END. */
3483         return actions_n + 1;
3484 }
3485
3486 /**
3487  * Check if the flow should be split due to hairpin.
3488  * The reason for the split is that in current HW we can't
3489  * support encap and push-vlan on Rx, so if a flow contains
3490  * these actions we move it to Tx.
3491  *
3492  * @param dev
3493  *   Pointer to Ethernet device.
3494  * @param[in] attr
3495  *   Flow rule attributes.
3496  * @param[in] actions
3497  *   Associated actions (list terminated by the END action).
3498  *
3499  * @return
3500  *   > 0 the number of actions and the flow should be split,
3501  *   0 when no split required.
3502  */
3503 static int
3504 flow_check_hairpin_split(struct rte_eth_dev *dev,
3505                          const struct rte_flow_attr *attr,
3506                          const struct rte_flow_action actions[])
3507 {
3508         int queue_action = 0;
3509         int action_n = 0;
3510         int split = 0;
3511         const struct rte_flow_action_queue *queue;
3512         const struct rte_flow_action_rss *rss;
3513         const struct rte_flow_action_raw_encap *raw_encap;
3514         const struct rte_eth_hairpin_conf *conf;
3515
3516         if (!attr->ingress)
3517                 return 0;
3518         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3519                 switch (actions->type) {
3520                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3521                         queue = actions->conf;
3522                         if (queue == NULL)
3523                                 return 0;
3524                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3525                         if (conf != NULL && !!conf->tx_explicit)
3526                                 return 0;
3527                         queue_action = 1;
3528                         action_n++;
3529                         break;
3530                 case RTE_FLOW_ACTION_TYPE_RSS:
3531                         rss = actions->conf;
3532                         if (rss == NULL || rss->queue_num == 0)
3533                                 return 0;
3534                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3535                         if (conf != NULL && !!conf->tx_explicit)
3536                                 return 0;
3537                         queue_action = 1;
3538                         action_n++;
3539                         break;
3540                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3541                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3542                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3543                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3544                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3545                         split++;
3546                         action_n++;
3547                         break;
3548                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3549                         raw_encap = actions->conf;
3550                         if (raw_encap->size >
3551                             (sizeof(struct rte_flow_item_eth) +
3552                              sizeof(struct rte_flow_item_ipv4)))
3553                                 split++;
3554                         action_n++;
3555                         break;
3556                 default:
3557                         action_n++;
3558                         break;
3559                 }
3560         }
3561         if (split && queue_action)
3562                 return action_n;
3563         return 0;
3564 }
3565
3566 /* Declare flow create/destroy prototype in advance. */
3567 static uint32_t
3568 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3569                  const struct rte_flow_attr *attr,
3570                  const struct rte_flow_item items[],
3571                  const struct rte_flow_action actions[],
3572                  bool external, struct rte_flow_error *error);
3573
3574 static void
3575 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3576                   uint32_t flow_idx);
3577
3578 struct mlx5_hlist_entry *
3579 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3580                        void *cb_ctx)
3581 {
3582         struct rte_eth_dev *dev = list->ctx;
3583         struct mlx5_priv *priv = dev->data->dev_private;
3584         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3585         struct mlx5_flow_mreg_copy_resource *mcp_res;
3586         struct rte_flow_error *error = ctx->error;
3587         uint32_t idx = 0;
3588         int ret;
3589         uint32_t mark_id = key;
3590         struct rte_flow_attr attr = {
3591                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3592                 .ingress = 1,
3593         };
3594         struct mlx5_rte_flow_item_tag tag_spec = {
3595                 .data = mark_id,
3596         };
3597         struct rte_flow_item items[] = {
3598                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3599         };
3600         struct rte_flow_action_mark ftag = {
3601                 .id = mark_id,
3602         };
3603         struct mlx5_flow_action_copy_mreg cp_mreg = {
3604                 .dst = REG_B,
3605                 .src = REG_NON,
3606         };
3607         struct rte_flow_action_jump jump = {
3608                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3609         };
3610         struct rte_flow_action actions[] = {
3611                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3612         };
3613
3614         /* Fill the register fileds in the flow. */
3615         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3616         if (ret < 0)
3617                 return NULL;
3618         tag_spec.id = ret;
3619         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3620         if (ret < 0)
3621                 return NULL;
3622         cp_mreg.src = ret;
3623         /* Provide the full width of FLAG specific value. */
3624         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3625                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3626         /* Build a new flow. */
3627         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3628                 items[0] = (struct rte_flow_item){
3629                         .type = (enum rte_flow_item_type)
3630                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3631                         .spec = &tag_spec,
3632                 };
3633                 items[1] = (struct rte_flow_item){
3634                         .type = RTE_FLOW_ITEM_TYPE_END,
3635                 };
3636                 actions[0] = (struct rte_flow_action){
3637                         .type = (enum rte_flow_action_type)
3638                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3639                         .conf = &ftag,
3640                 };
3641                 actions[1] = (struct rte_flow_action){
3642                         .type = (enum rte_flow_action_type)
3643                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3644                         .conf = &cp_mreg,
3645                 };
3646                 actions[2] = (struct rte_flow_action){
3647                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3648                         .conf = &jump,
3649                 };
3650                 actions[3] = (struct rte_flow_action){
3651                         .type = RTE_FLOW_ACTION_TYPE_END,
3652                 };
3653         } else {
3654                 /* Default rule, wildcard match. */
3655                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3656                 items[0] = (struct rte_flow_item){
3657                         .type = RTE_FLOW_ITEM_TYPE_END,
3658                 };
3659                 actions[0] = (struct rte_flow_action){
3660                         .type = (enum rte_flow_action_type)
3661                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3662                         .conf = &cp_mreg,
3663                 };
3664                 actions[1] = (struct rte_flow_action){
3665                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3666                         .conf = &jump,
3667                 };
3668                 actions[2] = (struct rte_flow_action){
3669                         .type = RTE_FLOW_ACTION_TYPE_END,
3670                 };
3671         }
3672         /* Build a new entry. */
3673         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3674         if (!mcp_res) {
3675                 rte_errno = ENOMEM;
3676                 return NULL;
3677         }
3678         mcp_res->idx = idx;
3679         /*
3680          * The copy Flows are not included in any list. There
3681          * ones are referenced from other Flows and can not
3682          * be applied, removed, deleted in ardbitrary order
3683          * by list traversing.
3684          */
3685         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3686                                          actions, false, error);
3687         if (!mcp_res->rix_flow) {
3688                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
3689                 return NULL;
3690         }
3691         return &mcp_res->hlist_ent;
3692 }
3693
3694 /**
3695  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3696  *
3697  * As mark_id is unique, if there's already a registered flow for the mark_id,
3698  * return by increasing the reference counter of the resource. Otherwise, create
3699  * the resource (mcp_res) and flow.
3700  *
3701  * Flow looks like,
3702  *   - If ingress port is ANY and reg_c[1] is mark_id,
3703  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3704  *
3705  * For default flow (zero mark_id), flow is like,
3706  *   - If ingress port is ANY,
3707  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3708  *
3709  * @param dev
3710  *   Pointer to Ethernet device.
3711  * @param mark_id
3712  *   ID of MARK action, zero means default flow for META.
3713  * @param[out] error
3714  *   Perform verbose error reporting if not NULL.
3715  *
3716  * @return
3717  *   Associated resource on success, NULL otherwise and rte_errno is set.
3718  */
3719 static struct mlx5_flow_mreg_copy_resource *
3720 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3721                           struct rte_flow_error *error)
3722 {
3723         struct mlx5_priv *priv = dev->data->dev_private;
3724         struct mlx5_hlist_entry *entry;
3725         struct mlx5_flow_cb_ctx ctx = {
3726                 .dev = dev,
3727                 .error = error,
3728         };
3729
3730         /* Check if already registered. */
3731         MLX5_ASSERT(priv->mreg_cp_tbl);
3732         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
3733         if (!entry)
3734                 return NULL;
3735         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
3736                             hlist_ent);
3737 }
3738
3739 void
3740 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
3741 {
3742         struct mlx5_flow_mreg_copy_resource *mcp_res =
3743                 container_of(entry, typeof(*mcp_res), hlist_ent);
3744         struct rte_eth_dev *dev = list->ctx;
3745         struct mlx5_priv *priv = dev->data->dev_private;
3746
3747         MLX5_ASSERT(mcp_res->rix_flow);
3748         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3749         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3750 }
3751
3752 /**
3753  * Release flow in RX_CP_TBL.
3754  *
3755  * @param dev
3756  *   Pointer to Ethernet device.
3757  * @flow
3758  *   Parent flow for wich copying is provided.
3759  */
3760 static void
3761 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3762                           struct rte_flow *flow)
3763 {
3764         struct mlx5_flow_mreg_copy_resource *mcp_res;
3765         struct mlx5_priv *priv = dev->data->dev_private;
3766
3767         if (!flow->rix_mreg_copy)
3768                 return;
3769         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3770                                  flow->rix_mreg_copy);
3771         if (!mcp_res || !priv->mreg_cp_tbl)
3772                 return;
3773         MLX5_ASSERT(mcp_res->rix_flow);
3774         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3775         flow->rix_mreg_copy = 0;
3776 }
3777
3778 /**
3779  * Remove the default copy action from RX_CP_TBL.
3780  *
3781  * This functions is called in the mlx5_dev_start(). No thread safe
3782  * is guaranteed.
3783  *
3784  * @param dev
3785  *   Pointer to Ethernet device.
3786  */
3787 static void
3788 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3789 {
3790         struct mlx5_hlist_entry *entry;
3791         struct mlx5_priv *priv = dev->data->dev_private;
3792
3793         /* Check if default flow is registered. */
3794         if (!priv->mreg_cp_tbl)
3795                 return;
3796         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
3797                                   MLX5_DEFAULT_COPY_ID, NULL);
3798         if (!entry)
3799                 return;
3800         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
3801 }
3802
3803 /**
3804  * Add the default copy action in in RX_CP_TBL.
3805  *
3806  * This functions is called in the mlx5_dev_start(). No thread safe
3807  * is guaranteed.
3808  *
3809  * @param dev
3810  *   Pointer to Ethernet device.
3811  * @param[out] error
3812  *   Perform verbose error reporting if not NULL.
3813  *
3814  * @return
3815  *   0 for success, negative value otherwise and rte_errno is set.
3816  */
3817 static int
3818 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3819                                   struct rte_flow_error *error)
3820 {
3821         struct mlx5_priv *priv = dev->data->dev_private;
3822         struct mlx5_flow_mreg_copy_resource *mcp_res;
3823
3824         /* Check whether extensive metadata feature is engaged. */
3825         if (!priv->config.dv_flow_en ||
3826             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3827             !mlx5_flow_ext_mreg_supported(dev) ||
3828             !priv->sh->dv_regc0_mask)
3829                 return 0;
3830         /*
3831          * Add default mreg copy flow may be called multiple time, but
3832          * only be called once in stop. Avoid register it twice.
3833          */
3834         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
3835                 return 0;
3836         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3837         if (!mcp_res)
3838                 return -rte_errno;
3839         return 0;
3840 }
3841
3842 /**
3843  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3844  *
3845  * All the flow having Q/RSS action should be split by
3846  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3847  * performs the following,
3848  *   - CQE->flow_tag := reg_c[1] (MARK)
3849  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3850  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3851  * but there should be a flow per each MARK ID set by MARK action.
3852  *
3853  * For the aforementioned reason, if there's a MARK action in flow's action
3854  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3855  * the MARK ID to CQE's flow_tag like,
3856  *   - If reg_c[1] is mark_id,
3857  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3858  *
3859  * For SET_META action which stores value in reg_c[0], as the destination is
3860  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3861  * MARK ID means the default flow. The default flow looks like,
3862  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3863  *
3864  * @param dev
3865  *   Pointer to Ethernet device.
3866  * @param flow
3867  *   Pointer to flow structure.
3868  * @param[in] actions
3869  *   Pointer to the list of actions.
3870  * @param[out] error
3871  *   Perform verbose error reporting if not NULL.
3872  *
3873  * @return
3874  *   0 on success, negative value otherwise and rte_errno is set.
3875  */
3876 static int
3877 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3878                             struct rte_flow *flow,
3879                             const struct rte_flow_action *actions,
3880                             struct rte_flow_error *error)
3881 {
3882         struct mlx5_priv *priv = dev->data->dev_private;
3883         struct mlx5_dev_config *config = &priv->config;
3884         struct mlx5_flow_mreg_copy_resource *mcp_res;
3885         const struct rte_flow_action_mark *mark;
3886
3887         /* Check whether extensive metadata feature is engaged. */
3888         if (!config->dv_flow_en ||
3889             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3890             !mlx5_flow_ext_mreg_supported(dev) ||
3891             !priv->sh->dv_regc0_mask)
3892                 return 0;
3893         /* Find MARK action. */
3894         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3895                 switch (actions->type) {
3896                 case RTE_FLOW_ACTION_TYPE_FLAG:
3897                         mcp_res = flow_mreg_add_copy_action
3898                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3899                         if (!mcp_res)
3900                                 return -rte_errno;
3901                         flow->rix_mreg_copy = mcp_res->idx;
3902                         return 0;
3903                 case RTE_FLOW_ACTION_TYPE_MARK:
3904                         mark = (const struct rte_flow_action_mark *)
3905                                 actions->conf;
3906                         mcp_res =
3907                                 flow_mreg_add_copy_action(dev, mark->id, error);
3908                         if (!mcp_res)
3909                                 return -rte_errno;
3910                         flow->rix_mreg_copy = mcp_res->idx;
3911                         return 0;
3912                 default:
3913                         break;
3914                 }
3915         }
3916         return 0;
3917 }
3918
3919 #define MLX5_MAX_SPLIT_ACTIONS 24
3920 #define MLX5_MAX_SPLIT_ITEMS 24
3921
3922 /**
3923  * Split the hairpin flow.
3924  * Since HW can't support encap and push-vlan on Rx, we move these
3925  * actions to Tx.
3926  * If the count action is after the encap then we also
3927  * move the count action. in this case the count will also measure
3928  * the outer bytes.
3929  *
3930  * @param dev
3931  *   Pointer to Ethernet device.
3932  * @param[in] actions
3933  *   Associated actions (list terminated by the END action).
3934  * @param[out] actions_rx
3935  *   Rx flow actions.
3936  * @param[out] actions_tx
3937  *   Tx flow actions..
3938  * @param[out] pattern_tx
3939  *   The pattern items for the Tx flow.
3940  * @param[out] flow_id
3941  *   The flow ID connected to this flow.
3942  *
3943  * @return
3944  *   0 on success.
3945  */
3946 static int
3947 flow_hairpin_split(struct rte_eth_dev *dev,
3948                    const struct rte_flow_action actions[],
3949                    struct rte_flow_action actions_rx[],
3950                    struct rte_flow_action actions_tx[],
3951                    struct rte_flow_item pattern_tx[],
3952                    uint32_t flow_id)
3953 {
3954         const struct rte_flow_action_raw_encap *raw_encap;
3955         const struct rte_flow_action_raw_decap *raw_decap;
3956         struct mlx5_rte_flow_action_set_tag *set_tag;
3957         struct rte_flow_action *tag_action;
3958         struct mlx5_rte_flow_item_tag *tag_item;
3959         struct rte_flow_item *item;
3960         char *addr;
3961         int encap = 0;
3962
3963         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3964                 switch (actions->type) {
3965                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3966                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3967                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3968                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3969                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3970                         rte_memcpy(actions_tx, actions,
3971                                sizeof(struct rte_flow_action));
3972                         actions_tx++;
3973                         break;
3974                 case RTE_FLOW_ACTION_TYPE_COUNT:
3975                         if (encap) {
3976                                 rte_memcpy(actions_tx, actions,
3977                                            sizeof(struct rte_flow_action));
3978                                 actions_tx++;
3979                         } else {
3980                                 rte_memcpy(actions_rx, actions,
3981                                            sizeof(struct rte_flow_action));
3982                                 actions_rx++;
3983                         }
3984                         break;
3985                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3986                         raw_encap = actions->conf;
3987                         if (raw_encap->size >
3988                             (sizeof(struct rte_flow_item_eth) +
3989                              sizeof(struct rte_flow_item_ipv4))) {
3990                                 memcpy(actions_tx, actions,
3991                                        sizeof(struct rte_flow_action));
3992                                 actions_tx++;
3993                                 encap = 1;
3994                         } else {
3995                                 rte_memcpy(actions_rx, actions,
3996                                            sizeof(struct rte_flow_action));
3997                                 actions_rx++;
3998                         }
3999                         break;
4000                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4001                         raw_decap = actions->conf;
4002                         if (raw_decap->size <
4003                             (sizeof(struct rte_flow_item_eth) +
4004                              sizeof(struct rte_flow_item_ipv4))) {
4005                                 memcpy(actions_tx, actions,
4006                                        sizeof(struct rte_flow_action));
4007                                 actions_tx++;
4008                         } else {
4009                                 rte_memcpy(actions_rx, actions,
4010                                            sizeof(struct rte_flow_action));
4011                                 actions_rx++;
4012                         }
4013                         break;
4014                 default:
4015                         rte_memcpy(actions_rx, actions,
4016                                    sizeof(struct rte_flow_action));
4017                         actions_rx++;
4018                         break;
4019                 }
4020         }
4021         /* Add set meta action and end action for the Rx flow. */
4022         tag_action = actions_rx;
4023         tag_action->type = (enum rte_flow_action_type)
4024                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4025         actions_rx++;
4026         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4027         actions_rx++;
4028         set_tag = (void *)actions_rx;
4029         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4030         MLX5_ASSERT(set_tag->id > REG_NON);
4031         set_tag->data = flow_id;
4032         tag_action->conf = set_tag;
4033         /* Create Tx item list. */
4034         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4035         addr = (void *)&pattern_tx[2];
4036         item = pattern_tx;
4037         item->type = (enum rte_flow_item_type)
4038                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4039         tag_item = (void *)addr;
4040         tag_item->data = flow_id;
4041         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4042         MLX5_ASSERT(set_tag->id > REG_NON);
4043         item->spec = tag_item;
4044         addr += sizeof(struct mlx5_rte_flow_item_tag);
4045         tag_item = (void *)addr;
4046         tag_item->data = UINT32_MAX;
4047         tag_item->id = UINT16_MAX;
4048         item->mask = tag_item;
4049         item->last = NULL;
4050         item++;
4051         item->type = RTE_FLOW_ITEM_TYPE_END;
4052         return 0;
4053 }
4054
4055 /**
4056  * The last stage of splitting chain, just creates the subflow
4057  * without any modification.
4058  *
4059  * @param[in] dev
4060  *   Pointer to Ethernet device.
4061  * @param[in] flow
4062  *   Parent flow structure pointer.
4063  * @param[in, out] sub_flow
4064  *   Pointer to return the created subflow, may be NULL.
4065  * @param[in] attr
4066  *   Flow rule attributes.
4067  * @param[in] items
4068  *   Pattern specification (list terminated by the END pattern item).
4069  * @param[in] actions
4070  *   Associated actions (list terminated by the END action).
4071  * @param[in] flow_split_info
4072  *   Pointer to flow split info structure.
4073  * @param[out] error
4074  *   Perform verbose error reporting if not NULL.
4075  * @return
4076  *   0 on success, negative value otherwise
4077  */
4078 static int
4079 flow_create_split_inner(struct rte_eth_dev *dev,
4080                         struct rte_flow *flow,
4081                         struct mlx5_flow **sub_flow,
4082                         const struct rte_flow_attr *attr,
4083                         const struct rte_flow_item items[],
4084                         const struct rte_flow_action actions[],
4085                         struct mlx5_flow_split_info *flow_split_info,
4086                         struct rte_flow_error *error)
4087 {
4088         struct mlx5_flow *dev_flow;
4089
4090         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4091                                     flow_split_info->flow_idx, error);
4092         if (!dev_flow)
4093                 return -rte_errno;
4094         dev_flow->flow = flow;
4095         dev_flow->external = flow_split_info->external;
4096         dev_flow->skip_scale = flow_split_info->skip_scale;
4097         /* Subflow object was created, we must include one in the list. */
4098         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4099                       dev_flow->handle, next);
4100         /*
4101          * If dev_flow is as one of the suffix flow, some actions in suffix
4102          * flow may need some user defined item layer flags, and pass the
4103          * Metadate rxq mark flag to suffix flow as well.
4104          */
4105         if (flow_split_info->prefix_layers)
4106                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4107         if (flow_split_info->prefix_mark)
4108                 dev_flow->handle->mark = 1;
4109         if (sub_flow)
4110                 *sub_flow = dev_flow;
4111         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4112 }
4113
4114 /**
4115  * Split the meter flow.
4116  *
4117  * As meter flow will split to three sub flow, other than meter
4118  * action, the other actions make sense to only meter accepts
4119  * the packet. If it need to be dropped, no other additional
4120  * actions should be take.
4121  *
4122  * One kind of special action which decapsulates the L3 tunnel
4123  * header will be in the prefix sub flow, as not to take the
4124  * L3 tunnel header into account.
4125  *
4126  * @param dev
4127  *   Pointer to Ethernet device.
4128  * @param[in] items
4129  *   Pattern specification (list terminated by the END pattern item).
4130  * @param[out] sfx_items
4131  *   Suffix flow match items (list terminated by the END pattern item).
4132  * @param[in] actions
4133  *   Associated actions (list terminated by the END action).
4134  * @param[out] actions_sfx
4135  *   Suffix flow actions.
4136  * @param[out] actions_pre
4137  *   Prefix flow actions.
4138  * @param[out] pattern_sfx
4139  *   The pattern items for the suffix flow.
4140  * @param[out] tag_sfx
4141  *   Pointer to suffix flow tag.
4142  *
4143  * @return
4144  *   0 on success.
4145  */
4146 static int
4147 flow_meter_split_prep(struct rte_eth_dev *dev,
4148                  const struct rte_flow_item items[],
4149                  struct rte_flow_item sfx_items[],
4150                  const struct rte_flow_action actions[],
4151                  struct rte_flow_action actions_sfx[],
4152                  struct rte_flow_action actions_pre[])
4153 {
4154         struct mlx5_priv *priv = dev->data->dev_private;
4155         struct rte_flow_action *tag_action = NULL;
4156         struct rte_flow_item *tag_item;
4157         struct mlx5_rte_flow_action_set_tag *set_tag;
4158         struct rte_flow_error error;
4159         const struct rte_flow_action_raw_encap *raw_encap;
4160         const struct rte_flow_action_raw_decap *raw_decap;
4161         struct mlx5_rte_flow_item_tag *tag_spec;
4162         struct mlx5_rte_flow_item_tag *tag_mask;
4163         uint32_t tag_id = 0;
4164         bool copy_vlan = false;
4165
4166         /* Prepare the actions for prefix and suffix flow. */
4167         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4168                 struct rte_flow_action **action_cur = NULL;
4169
4170                 switch (actions->type) {
4171                 case RTE_FLOW_ACTION_TYPE_METER:
4172                         /* Add the extra tag action first. */
4173                         tag_action = actions_pre;
4174                         tag_action->type = (enum rte_flow_action_type)
4175                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4176                         actions_pre++;
4177                         action_cur = &actions_pre;
4178                         break;
4179                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4180                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4181                         action_cur = &actions_pre;
4182                         break;
4183                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4184                         raw_encap = actions->conf;
4185                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4186                                 action_cur = &actions_pre;
4187                         break;
4188                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4189                         raw_decap = actions->conf;
4190                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4191                                 action_cur = &actions_pre;
4192                         break;
4193                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4194                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4195                         copy_vlan = true;
4196                         break;
4197                 default:
4198                         break;
4199                 }
4200                 if (!action_cur)
4201                         action_cur = &actions_sfx;
4202                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4203                 (*action_cur)++;
4204         }
4205         /* Add end action to the actions. */
4206         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4207         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4208         actions_pre++;
4209         /* Set the tag. */
4210         set_tag = (void *)actions_pre;
4211         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4212         mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4213                           &tag_id);
4214         if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
4215                 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
4216                         dev->data->port_id);
4217                 mlx5_ipool_free(priv->sh->ipool
4218                                 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
4219                 return 0;
4220         } else if (!tag_id) {
4221                 return 0;
4222         }
4223         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4224         assert(tag_action);
4225         tag_action->conf = set_tag;
4226         /* Prepare the suffix subflow items. */
4227         tag_item = sfx_items++;
4228         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4229                 int item_type = items->type;
4230
4231                 switch (item_type) {
4232                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4233                         memcpy(sfx_items, items, sizeof(*sfx_items));
4234                         sfx_items++;
4235                         break;
4236                 case RTE_FLOW_ITEM_TYPE_VLAN:
4237                         if (copy_vlan) {
4238                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4239                                 /*
4240                                  * Convert to internal match item, it is used
4241                                  * for vlan push and set vid.
4242                                  */
4243                                 sfx_items->type = (enum rte_flow_item_type)
4244                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4245                                 sfx_items++;
4246                         }
4247                         break;
4248                 default:
4249                         break;
4250                 }
4251         }
4252         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4253         sfx_items++;
4254         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4255         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4256         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4257         tag_mask = tag_spec + 1;
4258         tag_mask->data = 0xffffff00;
4259         tag_item->type = (enum rte_flow_item_type)
4260                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4261         tag_item->spec = tag_spec;
4262         tag_item->last = NULL;
4263         tag_item->mask = tag_mask;
4264         return tag_id;
4265 }
4266
4267 /**
4268  * Split action list having QUEUE/RSS for metadata register copy.
4269  *
4270  * Once Q/RSS action is detected in user's action list, the flow action
4271  * should be split in order to copy metadata registers, which will happen in
4272  * RX_CP_TBL like,
4273  *   - CQE->flow_tag := reg_c[1] (MARK)
4274  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4275  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4276  * This is because the last action of each flow must be a terminal action
4277  * (QUEUE, RSS or DROP).
4278  *
4279  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4280  * stored and kept in the mlx5_flow structure per each sub_flow.
4281  *
4282  * The Q/RSS action is replaced with,
4283  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4284  * And the following JUMP action is added at the end,
4285  *   - JUMP, to RX_CP_TBL.
4286  *
4287  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4288  * flow_create_split_metadata() routine. The flow will look like,
4289  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4290  *
4291  * @param dev
4292  *   Pointer to Ethernet device.
4293  * @param[out] split_actions
4294  *   Pointer to store split actions to jump to CP_TBL.
4295  * @param[in] actions
4296  *   Pointer to the list of original flow actions.
4297  * @param[in] qrss
4298  *   Pointer to the Q/RSS action.
4299  * @param[in] actions_n
4300  *   Number of original actions.
4301  * @param[out] error
4302  *   Perform verbose error reporting if not NULL.
4303  *
4304  * @return
4305  *   non-zero unique flow_id on success, otherwise 0 and
4306  *   error/rte_error are set.
4307  */
4308 static uint32_t
4309 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4310                           struct rte_flow_action *split_actions,
4311                           const struct rte_flow_action *actions,
4312                           const struct rte_flow_action *qrss,
4313                           int actions_n, struct rte_flow_error *error)
4314 {
4315         struct mlx5_priv *priv = dev->data->dev_private;
4316         struct mlx5_rte_flow_action_set_tag *set_tag;
4317         struct rte_flow_action_jump *jump;
4318         const int qrss_idx = qrss - actions;
4319         uint32_t flow_id = 0;
4320         int ret = 0;
4321
4322         /*
4323          * Given actions will be split
4324          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4325          * - Add jump to mreg CP_TBL.
4326          * As a result, there will be one more action.
4327          */
4328         ++actions_n;
4329         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4330         set_tag = (void *)(split_actions + actions_n);
4331         /*
4332          * If tag action is not set to void(it means we are not the meter
4333          * suffix flow), add the tag action. Since meter suffix flow already
4334          * has the tag added.
4335          */
4336         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4337                 /*
4338                  * Allocate the new subflow ID. This one is unique within
4339                  * device and not shared with representors. Otherwise,
4340                  * we would have to resolve multi-thread access synch
4341                  * issue. Each flow on the shared device is appended
4342                  * with source vport identifier, so the resulting
4343                  * flows will be unique in the shared (by master and
4344                  * representors) domain even if they have coinciding
4345                  * IDs.
4346                  */
4347                 mlx5_ipool_malloc(priv->sh->ipool
4348                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4349                 if (!flow_id)
4350                         return rte_flow_error_set(error, ENOMEM,
4351                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4352                                                   NULL, "can't allocate id "
4353                                                   "for split Q/RSS subflow");
4354                 /* Internal SET_TAG action to set flow ID. */
4355                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4356                         .data = flow_id,
4357                 };
4358                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4359                 if (ret < 0)
4360                         return ret;
4361                 set_tag->id = ret;
4362                 /* Construct new actions array. */
4363                 /* Replace QUEUE/RSS action. */
4364                 split_actions[qrss_idx] = (struct rte_flow_action){
4365                         .type = (enum rte_flow_action_type)
4366                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4367                         .conf = set_tag,
4368                 };
4369         }
4370         /* JUMP action to jump to mreg copy table (CP_TBL). */
4371         jump = (void *)(set_tag + 1);
4372         *jump = (struct rte_flow_action_jump){
4373                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4374         };
4375         split_actions[actions_n - 2] = (struct rte_flow_action){
4376                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4377                 .conf = jump,
4378         };
4379         split_actions[actions_n - 1] = (struct rte_flow_action){
4380                 .type = RTE_FLOW_ACTION_TYPE_END,
4381         };
4382         return flow_id;
4383 }
4384
4385 /**
4386  * Extend the given action list for Tx metadata copy.
4387  *
4388  * Copy the given action list to the ext_actions and add flow metadata register
4389  * copy action in order to copy reg_a set by WQE to reg_c[0].
4390  *
4391  * @param[out] ext_actions
4392  *   Pointer to the extended action list.
4393  * @param[in] actions
4394  *   Pointer to the list of actions.
4395  * @param[in] actions_n
4396  *   Number of actions in the list.
4397  * @param[out] error
4398  *   Perform verbose error reporting if not NULL.
4399  * @param[in] encap_idx
4400  *   The encap action inndex.
4401  *
4402  * @return
4403  *   0 on success, negative value otherwise
4404  */
4405 static int
4406 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4407                        struct rte_flow_action *ext_actions,
4408                        const struct rte_flow_action *actions,
4409                        int actions_n, struct rte_flow_error *error,
4410                        int encap_idx)
4411 {
4412         struct mlx5_flow_action_copy_mreg *cp_mreg =
4413                 (struct mlx5_flow_action_copy_mreg *)
4414                         (ext_actions + actions_n + 1);
4415         int ret;
4416
4417         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4418         if (ret < 0)
4419                 return ret;
4420         cp_mreg->dst = ret;
4421         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4422         if (ret < 0)
4423                 return ret;
4424         cp_mreg->src = ret;
4425         if (encap_idx != 0)
4426                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4427         if (encap_idx == actions_n - 1) {
4428                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4429                         .type = (enum rte_flow_action_type)
4430                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4431                         .conf = cp_mreg,
4432                 };
4433                 ext_actions[actions_n] = (struct rte_flow_action){
4434                         .type = RTE_FLOW_ACTION_TYPE_END,
4435                 };
4436         } else {
4437                 ext_actions[encap_idx] = (struct rte_flow_action){
4438                         .type = (enum rte_flow_action_type)
4439                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4440                         .conf = cp_mreg,
4441                 };
4442                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4443                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4444         }
4445         return 0;
4446 }
4447
4448 /**
4449  * Check the match action from the action list.
4450  *
4451  * @param[in] actions
4452  *   Pointer to the list of actions.
4453  * @param[in] attr
4454  *   Flow rule attributes.
4455  * @param[in] action
4456  *   The action to be check if exist.
4457  * @param[out] match_action_pos
4458  *   Pointer to the position of the matched action if exists, otherwise is -1.
4459  * @param[out] qrss_action_pos
4460  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4461  *
4462  * @return
4463  *   > 0 the total number of actions.
4464  *   0 if not found match action in action list.
4465  */
4466 static int
4467 flow_check_match_action(const struct rte_flow_action actions[],
4468                         const struct rte_flow_attr *attr,
4469                         enum rte_flow_action_type action,
4470                         int *match_action_pos, int *qrss_action_pos)
4471 {
4472         const struct rte_flow_action_sample *sample;
4473         int actions_n = 0;
4474         int jump_flag = 0;
4475         uint32_t ratio = 0;
4476         int sub_type = 0;
4477         int flag = 0;
4478
4479         *match_action_pos = -1;
4480         *qrss_action_pos = -1;
4481         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4482                 if (actions->type == action) {
4483                         flag = 1;
4484                         *match_action_pos = actions_n;
4485                 }
4486                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4487                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4488                         *qrss_action_pos = actions_n;
4489                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4490                         jump_flag = 1;
4491                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4492                         sample = actions->conf;
4493                         ratio = sample->ratio;
4494                         sub_type = ((const struct rte_flow_action *)
4495                                         (sample->actions))->type;
4496                 }
4497                 actions_n++;
4498         }
4499         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4500                 if (ratio == 1) {
4501                         /* JUMP Action not support for Mirroring;
4502                          * Mirroring support multi-destination;
4503                          */
4504                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4505                                 flag = 0;
4506                 }
4507         }
4508         /* Count RTE_FLOW_ACTION_TYPE_END. */
4509         return flag ? actions_n + 1 : 0;
4510 }
4511
4512 #define SAMPLE_SUFFIX_ITEM 2
4513
4514 /**
4515  * Split the sample flow.
4516  *
4517  * As sample flow will split to two sub flow, sample flow with
4518  * sample action, the other actions will move to new suffix flow.
4519  *
4520  * Also add unique tag id with tag action in the sample flow,
4521  * the same tag id will be as match in the suffix flow.
4522  *
4523  * @param dev
4524  *   Pointer to Ethernet device.
4525  * @param[in] fdb_tx
4526  *   FDB egress flow flag.
4527  * @param[out] sfx_items
4528  *   Suffix flow match items (list terminated by the END pattern item).
4529  * @param[in] actions
4530  *   Associated actions (list terminated by the END action).
4531  * @param[out] actions_sfx
4532  *   Suffix flow actions.
4533  * @param[out] actions_pre
4534  *   Prefix flow actions.
4535  * @param[in] actions_n
4536  *  The total number of actions.
4537  * @param[in] sample_action_pos
4538  *   The sample action position.
4539  * @param[in] qrss_action_pos
4540  *   The Queue/RSS action position.
4541  * @param[out] error
4542  *   Perform verbose error reporting if not NULL.
4543  *
4544  * @return
4545  *   0 on success, or unique flow_id, a negative errno value
4546  *   otherwise and rte_errno is set.
4547  */
4548 static int
4549 flow_sample_split_prep(struct rte_eth_dev *dev,
4550                        uint32_t fdb_tx,
4551                        struct rte_flow_item sfx_items[],
4552                        const struct rte_flow_action actions[],
4553                        struct rte_flow_action actions_sfx[],
4554                        struct rte_flow_action actions_pre[],
4555                        int actions_n,
4556                        int sample_action_pos,
4557                        int qrss_action_pos,
4558                        struct rte_flow_error *error)
4559 {
4560         struct mlx5_priv *priv = dev->data->dev_private;
4561         struct mlx5_rte_flow_action_set_tag *set_tag;
4562         struct mlx5_rte_flow_item_tag *tag_spec;
4563         struct mlx5_rte_flow_item_tag *tag_mask;
4564         uint32_t tag_id = 0;
4565         int index;
4566         int ret;
4567
4568         if (sample_action_pos < 0)
4569                 return rte_flow_error_set(error, EINVAL,
4570                                           RTE_FLOW_ERROR_TYPE_ACTION,
4571                                           NULL, "invalid position of sample "
4572                                           "action in list");
4573         if (!fdb_tx) {
4574                 /* Prepare the prefix tag action. */
4575                 set_tag = (void *)(actions_pre + actions_n + 1);
4576                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4577                 if (ret < 0)
4578                         return ret;
4579                 set_tag->id = ret;
4580                 mlx5_ipool_malloc(priv->sh->ipool
4581                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
4582                 set_tag->data = tag_id;
4583                 /* Prepare the suffix subflow items. */
4584                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4585                 tag_spec->data = tag_id;
4586                 tag_spec->id = set_tag->id;
4587                 tag_mask = tag_spec + 1;
4588                 tag_mask->data = UINT32_MAX;
4589                 sfx_items[0] = (struct rte_flow_item){
4590                         .type = (enum rte_flow_item_type)
4591                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4592                         .spec = tag_spec,
4593                         .last = NULL,
4594                         .mask = tag_mask,
4595                 };
4596                 sfx_items[1] = (struct rte_flow_item){
4597                         .type = (enum rte_flow_item_type)
4598                                 RTE_FLOW_ITEM_TYPE_END,
4599                 };
4600         }
4601         /* Prepare the actions for prefix and suffix flow. */
4602         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4603                 index = qrss_action_pos;
4604                 /* Put the preceding the Queue/RSS action into prefix flow. */
4605                 if (index != 0)
4606                         memcpy(actions_pre, actions,
4607                                sizeof(struct rte_flow_action) * index);
4608                 /* Put others preceding the sample action into prefix flow. */
4609                 if (sample_action_pos > index + 1)
4610                         memcpy(actions_pre + index, actions + index + 1,
4611                                sizeof(struct rte_flow_action) *
4612                                (sample_action_pos - index - 1));
4613                 index = sample_action_pos - 1;
4614                 /* Put Queue/RSS action into Suffix flow. */
4615                 memcpy(actions_sfx, actions + qrss_action_pos,
4616                        sizeof(struct rte_flow_action));
4617                 actions_sfx++;
4618         } else {
4619                 index = sample_action_pos;
4620                 if (index != 0)
4621                         memcpy(actions_pre, actions,
4622                                sizeof(struct rte_flow_action) * index);
4623         }
4624         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4625         if (!fdb_tx) {
4626                 actions_pre[index++] =
4627                         (struct rte_flow_action){
4628                         .type = (enum rte_flow_action_type)
4629                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4630                         .conf = set_tag,
4631                 };
4632         }
4633         memcpy(actions_pre + index, actions + sample_action_pos,
4634                sizeof(struct rte_flow_action));
4635         index += 1;
4636         actions_pre[index] = (struct rte_flow_action){
4637                 .type = (enum rte_flow_action_type)
4638                         RTE_FLOW_ACTION_TYPE_END,
4639         };
4640         /* Put the actions after sample into Suffix flow. */
4641         memcpy(actions_sfx, actions + sample_action_pos + 1,
4642                sizeof(struct rte_flow_action) *
4643                (actions_n - sample_action_pos - 1));
4644         return tag_id;
4645 }
4646
4647 /**
4648  * The splitting for metadata feature.
4649  *
4650  * - Q/RSS action on NIC Rx should be split in order to pass by
4651  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4652  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4653  *
4654  * - All the actions on NIC Tx should have a mreg copy action to
4655  *   copy reg_a from WQE to reg_c[0].
4656  *
4657  * @param dev
4658  *   Pointer to Ethernet device.
4659  * @param[in] flow
4660  *   Parent flow structure pointer.
4661  * @param[in] attr
4662  *   Flow rule attributes.
4663  * @param[in] items
4664  *   Pattern specification (list terminated by the END pattern item).
4665  * @param[in] actions
4666  *   Associated actions (list terminated by the END action).
4667  * @param[in] flow_split_info
4668  *   Pointer to flow split info structure.
4669  * @param[out] error
4670  *   Perform verbose error reporting if not NULL.
4671  * @return
4672  *   0 on success, negative value otherwise
4673  */
4674 static int
4675 flow_create_split_metadata(struct rte_eth_dev *dev,
4676                            struct rte_flow *flow,
4677                            const struct rte_flow_attr *attr,
4678                            const struct rte_flow_item items[],
4679                            const struct rte_flow_action actions[],
4680                            struct mlx5_flow_split_info *flow_split_info,
4681                            struct rte_flow_error *error)
4682 {
4683         struct mlx5_priv *priv = dev->data->dev_private;
4684         struct mlx5_dev_config *config = &priv->config;
4685         const struct rte_flow_action *qrss = NULL;
4686         struct rte_flow_action *ext_actions = NULL;
4687         struct mlx5_flow *dev_flow = NULL;
4688         uint32_t qrss_id = 0;
4689         int mtr_sfx = 0;
4690         size_t act_size;
4691         int actions_n;
4692         int encap_idx;
4693         int ret;
4694
4695         /* Check whether extensive metadata feature is engaged. */
4696         if (!config->dv_flow_en ||
4697             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4698             !mlx5_flow_ext_mreg_supported(dev))
4699                 return flow_create_split_inner(dev, flow, NULL, attr, items,
4700                                                actions, flow_split_info, error);
4701         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4702                                                            &encap_idx);
4703         if (qrss) {
4704                 /* Exclude hairpin flows from splitting. */
4705                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4706                         const struct rte_flow_action_queue *queue;
4707
4708                         queue = qrss->conf;
4709                         if (mlx5_rxq_get_type(dev, queue->index) ==
4710                             MLX5_RXQ_TYPE_HAIRPIN)
4711                                 qrss = NULL;
4712                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4713                         const struct rte_flow_action_rss *rss;
4714
4715                         rss = qrss->conf;
4716                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4717                             MLX5_RXQ_TYPE_HAIRPIN)
4718                                 qrss = NULL;
4719                 }
4720         }
4721         if (qrss) {
4722                 /* Check if it is in meter suffix table. */
4723                 mtr_sfx = attr->group == (attr->transfer ?
4724                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4725                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4726                 /*
4727                  * Q/RSS action on NIC Rx should be split in order to pass by
4728                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4729                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4730                  */
4731                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4732                            sizeof(struct rte_flow_action_set_tag) +
4733                            sizeof(struct rte_flow_action_jump);
4734                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4735                                           SOCKET_ID_ANY);
4736                 if (!ext_actions)
4737                         return rte_flow_error_set(error, ENOMEM,
4738                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4739                                                   NULL, "no memory to split "
4740                                                   "metadata flow");
4741                 /*
4742                  * If we are the suffix flow of meter, tag already exist.
4743                  * Set the tag action to void.
4744                  */
4745                 if (mtr_sfx)
4746                         ext_actions[qrss - actions].type =
4747                                                 RTE_FLOW_ACTION_TYPE_VOID;
4748                 else
4749                         ext_actions[qrss - actions].type =
4750                                                 (enum rte_flow_action_type)
4751                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4752                 /*
4753                  * Create the new actions list with removed Q/RSS action
4754                  * and appended set tag and jump to register copy table
4755                  * (RX_CP_TBL). We should preallocate unique tag ID here
4756                  * in advance, because it is needed for set tag action.
4757                  */
4758                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4759                                                     qrss, actions_n, error);
4760                 if (!mtr_sfx && !qrss_id) {
4761                         ret = -rte_errno;
4762                         goto exit;
4763                 }
4764         } else if (attr->egress && !attr->transfer) {
4765                 /*
4766                  * All the actions on NIC Tx should have a metadata register
4767                  * copy action to copy reg_a from WQE to reg_c[meta]
4768                  */
4769                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4770                            sizeof(struct mlx5_flow_action_copy_mreg);
4771                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4772                                           SOCKET_ID_ANY);
4773                 if (!ext_actions)
4774                         return rte_flow_error_set(error, ENOMEM,
4775                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4776                                                   NULL, "no memory to split "
4777                                                   "metadata flow");
4778                 /* Create the action list appended with copy register. */
4779                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4780                                              actions_n, error, encap_idx);
4781                 if (ret < 0)
4782                         goto exit;
4783         }
4784         /* Add the unmodified original or prefix subflow. */
4785         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
4786                                       items, ext_actions ? ext_actions :
4787                                       actions, flow_split_info, error);
4788         if (ret < 0)
4789                 goto exit;
4790         MLX5_ASSERT(dev_flow);
4791         if (qrss) {
4792                 const struct rte_flow_attr q_attr = {
4793                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4794                         .ingress = 1,
4795                 };
4796                 /* Internal PMD action to set register. */
4797                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4798                         .data = qrss_id,
4799                         .id = REG_NON,
4800                 };
4801                 struct rte_flow_item q_items[] = {
4802                         {
4803                                 .type = (enum rte_flow_item_type)
4804                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4805                                 .spec = &q_tag_spec,
4806                                 .last = NULL,
4807                                 .mask = NULL,
4808                         },
4809                         {
4810                                 .type = RTE_FLOW_ITEM_TYPE_END,
4811                         },
4812                 };
4813                 struct rte_flow_action q_actions[] = {
4814                         {
4815                                 .type = qrss->type,
4816                                 .conf = qrss->conf,
4817                         },
4818                         {
4819                                 .type = RTE_FLOW_ACTION_TYPE_END,
4820                         },
4821                 };
4822                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4823
4824                 /*
4825                  * Configure the tag item only if there is no meter subflow.
4826                  * Since tag is already marked in the meter suffix subflow
4827                  * we can just use the meter suffix items as is.
4828                  */
4829                 if (qrss_id) {
4830                         /* Not meter subflow. */
4831                         MLX5_ASSERT(!mtr_sfx);
4832                         /*
4833                          * Put unique id in prefix flow due to it is destroyed
4834                          * after suffix flow and id will be freed after there
4835                          * is no actual flows with this id and identifier
4836                          * reallocation becomes possible (for example, for
4837                          * other flows in other threads).
4838                          */
4839                         dev_flow->handle->split_flow_id = qrss_id;
4840                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4841                                                    error);
4842                         if (ret < 0)
4843                                 goto exit;
4844                         q_tag_spec.id = ret;
4845                 }
4846                 dev_flow = NULL;
4847                 /* Add suffix subflow to execute Q/RSS. */
4848                 flow_split_info->prefix_layers = layers;
4849                 flow_split_info->prefix_mark = 0;
4850                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4851                                               &q_attr, mtr_sfx ? items :
4852                                               q_items, q_actions,
4853                                               flow_split_info, error);
4854                 if (ret < 0)
4855                         goto exit;
4856                 /* qrss ID should be freed if failed. */
4857                 qrss_id = 0;
4858                 MLX5_ASSERT(dev_flow);
4859         }
4860
4861 exit:
4862         /*
4863          * We do not destroy the partially created sub_flows in case of error.
4864          * These ones are included into parent flow list and will be destroyed
4865          * by flow_drv_destroy.
4866          */
4867         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4868                         qrss_id);
4869         mlx5_free(ext_actions);
4870         return ret;
4871 }
4872
4873 /**
4874  * The splitting for meter feature.
4875  *
4876  * - The meter flow will be split to two flows as prefix and
4877  *   suffix flow. The packets make sense only it pass the prefix
4878  *   meter action.
4879  *
4880  * - Reg_C_5 is used for the packet to match betweend prefix and
4881  *   suffix flow.
4882  *
4883  * @param dev
4884  *   Pointer to Ethernet device.
4885  * @param[in] flow
4886  *   Parent flow structure pointer.
4887  * @param[in] attr
4888  *   Flow rule attributes.
4889  * @param[in] items
4890  *   Pattern specification (list terminated by the END pattern item).
4891  * @param[in] actions
4892  *   Associated actions (list terminated by the END action).
4893  * @param[in] flow_split_info
4894  *   Pointer to flow split info structure.
4895  * @param[out] error
4896  *   Perform verbose error reporting if not NULL.
4897  * @return
4898  *   0 on success, negative value otherwise
4899  */
4900 static int
4901 flow_create_split_meter(struct rte_eth_dev *dev,
4902                         struct rte_flow *flow,
4903                         const struct rte_flow_attr *attr,
4904                         const struct rte_flow_item items[],
4905                         const struct rte_flow_action actions[],
4906                         struct mlx5_flow_split_info *flow_split_info,
4907                         struct rte_flow_error *error)
4908 {
4909         struct mlx5_priv *priv = dev->data->dev_private;
4910         struct rte_flow_action *sfx_actions = NULL;
4911         struct rte_flow_action *pre_actions = NULL;
4912         struct rte_flow_item *sfx_items = NULL;
4913         struct mlx5_flow *dev_flow = NULL;
4914         struct rte_flow_attr sfx_attr = *attr;
4915         uint32_t mtr = 0;
4916         uint32_t mtr_tag_id = 0;
4917         size_t act_size;
4918         size_t item_size;
4919         int actions_n = 0;
4920         int ret;
4921
4922         if (priv->mtr_en)
4923                 actions_n = flow_check_meter_action(actions, &mtr);
4924         if (mtr) {
4925                 /* The five prefix actions: meter, decap, encap, tag, end. */
4926                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4927                            sizeof(struct mlx5_rte_flow_action_set_tag);
4928                 /* tag, vlan, port id, end. */
4929 #define METER_SUFFIX_ITEM 4
4930                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4931                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4932                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4933                                           0, SOCKET_ID_ANY);
4934                 if (!sfx_actions)
4935                         return rte_flow_error_set(error, ENOMEM,
4936                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4937                                                   NULL, "no memory to split "
4938                                                   "meter flow");
4939                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4940                              act_size);
4941                 pre_actions = sfx_actions + actions_n;
4942                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4943                                                    actions, sfx_actions,
4944                                                    pre_actions);
4945                 if (!mtr_tag_id) {
4946                         ret = -rte_errno;
4947                         goto exit;
4948                 }
4949                 /* Add the prefix subflow. */
4950                 flow_split_info->prefix_mark = 0;
4951                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4952                                               attr, items, pre_actions,
4953                                               flow_split_info, error);
4954                 if (ret) {
4955                         ret = -rte_errno;
4956                         goto exit;
4957                 }
4958                 dev_flow->handle->split_flow_id = mtr_tag_id;
4959                 /* Setting the sfx group atrr. */
4960                 sfx_attr.group = sfx_attr.transfer ?
4961                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4962                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4963                 flow_split_info->prefix_layers =
4964                                 flow_get_prefix_layer_flags(dev_flow);
4965                 flow_split_info->prefix_mark = dev_flow->handle->mark;
4966         }
4967         /* Add the prefix subflow. */
4968         ret = flow_create_split_metadata(dev, flow,
4969                                          &sfx_attr, sfx_items ?
4970                                          sfx_items : items,
4971                                          sfx_actions ? sfx_actions : actions,
4972                                          flow_split_info, error);
4973 exit:
4974         if (sfx_actions)
4975                 mlx5_free(sfx_actions);
4976         return ret;
4977 }
4978
4979 /**
4980  * The splitting for sample feature.
4981  *
4982  * Once Sample action is detected in the action list, the flow actions should
4983  * be split into prefix sub flow and suffix sub flow.
4984  *
4985  * The original items remain in the prefix sub flow, all actions preceding the
4986  * sample action and the sample action itself will be copied to the prefix
4987  * sub flow, the actions following the sample action will be copied to the
4988  * suffix sub flow, Queue action always be located in the suffix sub flow.
4989  *
4990  * In order to make the packet from prefix sub flow matches with suffix sub
4991  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
4992  * flow uses tag item with the unique flow id.
4993  *
4994  * @param dev
4995  *   Pointer to Ethernet device.
4996  * @param[in] flow
4997  *   Parent flow structure pointer.
4998  * @param[in] attr
4999  *   Flow rule attributes.
5000  * @param[in] items
5001  *   Pattern specification (list terminated by the END pattern item).
5002  * @param[in] actions
5003  *   Associated actions (list terminated by the END action).
5004  * @param[in] flow_split_info
5005  *   Pointer to flow split info structure.
5006  * @param[out] error
5007  *   Perform verbose error reporting if not NULL.
5008  * @return
5009  *   0 on success, negative value otherwise
5010  */
5011 static int
5012 flow_create_split_sample(struct rte_eth_dev *dev,
5013                          struct rte_flow *flow,
5014                          const struct rte_flow_attr *attr,
5015                          const struct rte_flow_item items[],
5016                          const struct rte_flow_action actions[],
5017                          struct mlx5_flow_split_info *flow_split_info,
5018                          struct rte_flow_error *error)
5019 {
5020         struct mlx5_priv *priv = dev->data->dev_private;
5021         struct rte_flow_action *sfx_actions = NULL;
5022         struct rte_flow_action *pre_actions = NULL;
5023         struct rte_flow_item *sfx_items = NULL;
5024         struct mlx5_flow *dev_flow = NULL;
5025         struct rte_flow_attr sfx_attr = *attr;
5026 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5027         struct mlx5_flow_dv_sample_resource *sample_res;
5028         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5029         struct mlx5_flow_tbl_resource *sfx_tbl;
5030         union mlx5_flow_tbl_key sfx_table_key;
5031 #endif
5032         size_t act_size;
5033         size_t item_size;
5034         uint32_t fdb_tx = 0;
5035         int32_t tag_id = 0;
5036         int actions_n = 0;
5037         int sample_action_pos;
5038         int qrss_action_pos;
5039         int ret = 0;
5040
5041         if (priv->sampler_en)
5042                 actions_n = flow_check_match_action(actions, attr,
5043                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5044                                         &sample_action_pos, &qrss_action_pos);
5045         if (actions_n) {
5046                 /* The prefix actions must includes sample, tag, end. */
5047                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5048                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5049                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5050                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5051                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5052                                           item_size), 0, SOCKET_ID_ANY);
5053                 if (!sfx_actions)
5054                         return rte_flow_error_set(error, ENOMEM,
5055                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5056                                                   NULL, "no memory to split "
5057                                                   "sample flow");
5058                 /* The representor_id is -1 for uplink. */
5059                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5060                 if (!fdb_tx)
5061                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5062                                         + act_size);
5063                 pre_actions = sfx_actions + actions_n;
5064                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5065                                                 actions, sfx_actions,
5066                                                 pre_actions, actions_n,
5067                                                 sample_action_pos,
5068                                                 qrss_action_pos, error);
5069                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5070                         ret = -rte_errno;
5071                         goto exit;
5072                 }
5073                 /* Add the prefix subflow. */
5074                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5075                                               items, pre_actions,
5076                                               flow_split_info, error);
5077                 if (ret) {
5078                         ret = -rte_errno;
5079                         goto exit;
5080                 }
5081                 dev_flow->handle->split_flow_id = tag_id;
5082 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5083                 /* Set the sfx group attr. */
5084                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5085                                         dev_flow->dv.sample_res;
5086                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5087                                         sample_res->normal_path_tbl;
5088                 sfx_tbl_data = container_of(sfx_tbl,
5089                                         struct mlx5_flow_tbl_data_entry, tbl);
5090                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5091                 sfx_attr.group = sfx_attr.transfer ?
5092                                         (sfx_table_key.table_id - 1) :
5093                                          sfx_table_key.table_id;
5094                 flow_split_info->prefix_layers =
5095                                 flow_get_prefix_layer_flags(dev_flow);
5096                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5097                 /* Suffix group level already be scaled with factor, set
5098                  * skip_scale to 1 to avoid scale again in translation.
5099                  */
5100                 flow_split_info->skip_scale = 1;
5101 #endif
5102         }
5103         /* Add the suffix subflow. */
5104         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5105                                       sfx_items ? sfx_items : items,
5106                                       sfx_actions ? sfx_actions : actions,
5107                                       flow_split_info, error);
5108 exit:
5109         if (sfx_actions)
5110                 mlx5_free(sfx_actions);
5111         return ret;
5112 }
5113
5114 /**
5115  * Split the flow to subflow set. The splitters might be linked
5116  * in the chain, like this:
5117  * flow_create_split_outer() calls:
5118  *   flow_create_split_meter() calls:
5119  *     flow_create_split_metadata(meter_subflow_0) calls:
5120  *       flow_create_split_inner(metadata_subflow_0)
5121  *       flow_create_split_inner(metadata_subflow_1)
5122  *       flow_create_split_inner(metadata_subflow_2)
5123  *     flow_create_split_metadata(meter_subflow_1) calls:
5124  *       flow_create_split_inner(metadata_subflow_0)
5125  *       flow_create_split_inner(metadata_subflow_1)
5126  *       flow_create_split_inner(metadata_subflow_2)
5127  *
5128  * This provide flexible way to add new levels of flow splitting.
5129  * The all of successfully created subflows are included to the
5130  * parent flow dev_flow list.
5131  *
5132  * @param dev
5133  *   Pointer to Ethernet device.
5134  * @param[in] flow
5135  *   Parent flow structure pointer.
5136  * @param[in] attr
5137  *   Flow rule attributes.
5138  * @param[in] items
5139  *   Pattern specification (list terminated by the END pattern item).
5140  * @param[in] actions
5141  *   Associated actions (list terminated by the END action).
5142  * @param[in] flow_split_info
5143  *   Pointer to flow split info structure.
5144  * @param[out] error
5145  *   Perform verbose error reporting if not NULL.
5146  * @return
5147  *   0 on success, negative value otherwise
5148  */
5149 static int
5150 flow_create_split_outer(struct rte_eth_dev *dev,
5151                         struct rte_flow *flow,
5152                         const struct rte_flow_attr *attr,
5153                         const struct rte_flow_item items[],
5154                         const struct rte_flow_action actions[],
5155                         struct mlx5_flow_split_info *flow_split_info,
5156                         struct rte_flow_error *error)
5157 {
5158         int ret;
5159
5160         ret = flow_create_split_sample(dev, flow, attr, items,
5161                                        actions, flow_split_info, error);
5162         MLX5_ASSERT(ret <= 0);
5163         return ret;
5164 }
5165
5166 static struct mlx5_flow_tunnel *
5167 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5168                       const struct rte_flow_attr *attr,
5169                       const struct rte_flow_item items[],
5170                       const struct rte_flow_action actions[])
5171 {
5172         struct mlx5_flow_tunnel *tunnel;
5173
5174 #pragma GCC diagnostic push
5175 #pragma GCC diagnostic ignored "-Wcast-qual"
5176         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5177                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5178         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5179                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5180         else
5181                 tunnel = NULL;
5182 #pragma GCC diagnostic pop
5183
5184         return tunnel;
5185 }
5186
5187 /**
5188  * Adjust flow RSS workspace if needed.
5189  *
5190  * @param wks
5191  *   Pointer to thread flow work space.
5192  * @param rss_desc
5193  *   Pointer to RSS descriptor.
5194  * @param[in] nrssq_num
5195  *   New RSS queue number.
5196  *
5197  * @return
5198  *   0 on success, -1 otherwise and rte_errno is set.
5199  */
5200 static int
5201 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
5202                           struct mlx5_flow_rss_desc *rss_desc,
5203                           uint32_t nrssq_num)
5204 {
5205         if (likely(nrssq_num <= wks->rssq_num))
5206                 return 0;
5207         rss_desc->queue = realloc(rss_desc->queue,
5208                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
5209         if (!rss_desc->queue) {
5210                 rte_errno = ENOMEM;
5211                 return -1;
5212         }
5213         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
5214         return 0;
5215 }
5216
5217 /**
5218  * Create a flow and add it to @p list.
5219  *
5220  * @param dev
5221  *   Pointer to Ethernet device.
5222  * @param list
5223  *   Pointer to a TAILQ flow list. If this parameter NULL,
5224  *   no list insertion occurred, flow is just created,
5225  *   this is caller's responsibility to track the
5226  *   created flow.
5227  * @param[in] attr
5228  *   Flow rule attributes.
5229  * @param[in] items
5230  *   Pattern specification (list terminated by the END pattern item).
5231  * @param[in] actions
5232  *   Associated actions (list terminated by the END action).
5233  * @param[in] external
5234  *   This flow rule is created by request external to PMD.
5235  * @param[out] error
5236  *   Perform verbose error reporting if not NULL.
5237  *
5238  * @return
5239  *   A flow index on success, 0 otherwise and rte_errno is set.
5240  */
5241 static uint32_t
5242 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5243                  const struct rte_flow_attr *attr,
5244                  const struct rte_flow_item items[],
5245                  const struct rte_flow_action original_actions[],
5246                  bool external, struct rte_flow_error *error)
5247 {
5248         struct mlx5_priv *priv = dev->data->dev_private;
5249         struct rte_flow *flow = NULL;
5250         struct mlx5_flow *dev_flow;
5251         const struct rte_flow_action_rss *rss;
5252         struct mlx5_translated_shared_action
5253                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5254         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5255         union {
5256                 struct mlx5_flow_expand_rss buf;
5257                 uint8_t buffer[2048];
5258         } expand_buffer;
5259         union {
5260                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5261                 uint8_t buffer[2048];
5262         } actions_rx;
5263         union {
5264                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5265                 uint8_t buffer[2048];
5266         } actions_hairpin_tx;
5267         union {
5268                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5269                 uint8_t buffer[2048];
5270         } items_tx;
5271         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5272         struct mlx5_flow_rss_desc *rss_desc;
5273         const struct rte_flow_action *p_actions_rx;
5274         uint32_t i;
5275         uint32_t idx = 0;
5276         int hairpin_flow;
5277         struct rte_flow_attr attr_tx = { .priority = 0 };
5278         const struct rte_flow_action *actions;
5279         struct rte_flow_action *translated_actions = NULL;
5280         struct mlx5_flow_tunnel *tunnel;
5281         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5282         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
5283         struct mlx5_flow_split_info flow_split_info = {
5284                 .external = !!external,
5285                 .skip_scale = 0,
5286                 .flow_idx = 0,
5287                 .prefix_mark = 0,
5288                 .prefix_layers = 0
5289         };
5290         int ret;
5291
5292         MLX5_ASSERT(wks);
5293         rss_desc = &wks->rss_desc;
5294         ret = flow_shared_actions_translate(dev, original_actions,
5295                                             shared_actions,
5296                                             &shared_actions_n,
5297                                             &translated_actions, error);
5298         if (ret < 0) {
5299                 MLX5_ASSERT(translated_actions == NULL);
5300                 return 0;
5301         }
5302         actions = translated_actions ? translated_actions : original_actions;
5303         p_actions_rx = actions;
5304         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5305         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
5306                                 external, hairpin_flow, error);
5307         if (ret < 0)
5308                 goto error_before_hairpin_split;
5309         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5310         if (!flow) {
5311                 rte_errno = ENOMEM;
5312                 goto error_before_hairpin_split;
5313         }
5314         if (hairpin_flow > 0) {
5315                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5316                         rte_errno = EINVAL;
5317                         goto error_before_hairpin_split;
5318                 }
5319                 flow_hairpin_split(dev, actions, actions_rx.actions,
5320                                    actions_hairpin_tx.actions, items_tx.items,
5321                                    idx);
5322                 p_actions_rx = actions_rx.actions;
5323         }
5324         flow_split_info.flow_idx = idx;
5325         flow->drv_type = flow_get_drv_type(dev, attr);
5326         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5327                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5328         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
5329         rss = flow_get_rss_action(p_actions_rx);
5330         if (rss) {
5331                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
5332                         return 0;
5333                 /*
5334                  * The following information is required by
5335                  * mlx5_flow_hashfields_adjust() in advance.
5336                  */
5337                 rss_desc->level = rss->level;
5338                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5339                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5340         }
5341         flow->dev_handles = 0;
5342         if (rss && rss->types) {
5343                 unsigned int graph_root;
5344
5345                 graph_root = find_graph_root(items, rss->level);
5346                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5347                                            items, rss->types,
5348                                            mlx5_support_expansion, graph_root);
5349                 MLX5_ASSERT(ret > 0 &&
5350                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5351         } else {
5352                 buf->entries = 1;
5353                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5354         }
5355         rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions,
5356                                                       shared_actions_n);
5357         for (i = 0; i < buf->entries; ++i) {
5358                 /* Initialize flow split data. */
5359                 flow_split_info.prefix_layers = 0;
5360                 flow_split_info.prefix_mark = 0;
5361                 flow_split_info.skip_scale = 0;
5362                 /*
5363                  * The splitter may create multiple dev_flows,
5364                  * depending on configuration. In the simplest
5365                  * case it just creates unmodified original flow.
5366                  */
5367                 ret = flow_create_split_outer(dev, flow, attr,
5368                                               buf->entry[i].pattern,
5369                                               p_actions_rx, &flow_split_info,
5370                                               error);
5371                 if (ret < 0)
5372                         goto error;
5373                 if (is_flow_tunnel_steer_rule(dev, attr,
5374                                               buf->entry[i].pattern,
5375                                               p_actions_rx)) {
5376                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5377                                                            p_actions_rx,
5378                                                            idx,
5379                                                            &default_miss_ctx,
5380                                                            error);
5381                         if (ret < 0) {
5382                                 mlx5_free(default_miss_ctx.queue);
5383                                 goto error;
5384                         }
5385                 }
5386         }
5387         /* Create the tx flow. */
5388         if (hairpin_flow) {
5389                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5390                 attr_tx.ingress = 0;
5391                 attr_tx.egress = 1;
5392                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5393                                          actions_hairpin_tx.actions,
5394                                          idx, error);
5395                 if (!dev_flow)
5396                         goto error;
5397                 dev_flow->flow = flow;
5398                 dev_flow->external = 0;
5399                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5400                               dev_flow->handle, next);
5401                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5402                                          items_tx.items,
5403                                          actions_hairpin_tx.actions, error);
5404                 if (ret < 0)
5405                         goto error;
5406         }
5407         /*
5408          * Update the metadata register copy table. If extensive
5409          * metadata feature is enabled and registers are supported
5410          * we might create the extra rte_flow for each unique
5411          * MARK/FLAG action ID.
5412          *
5413          * The table is updated for ingress Flows only, because
5414          * the egress Flows belong to the different device and
5415          * copy table should be updated in peer NIC Rx domain.
5416          */
5417         if (attr->ingress &&
5418             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5419                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5420                 if (ret)
5421                         goto error;
5422         }
5423         /*
5424          * If the flow is external (from application) OR device is started,
5425          * OR mreg discover, then apply immediately.
5426          */
5427         if (external || dev->data->dev_started ||
5428             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
5429              attr->priority == MLX5_FLOW_PRIO_RSVD)) {
5430                 ret = flow_drv_apply(dev, flow, error);
5431                 if (ret < 0)
5432                         goto error;
5433         }
5434         if (list) {
5435                 rte_spinlock_lock(&priv->flow_list_lock);
5436                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5437                              flow, next);
5438                 rte_spinlock_unlock(&priv->flow_list_lock);
5439         }
5440         flow_rxq_flags_set(dev, flow);
5441         rte_free(translated_actions);
5442         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5443         if (tunnel) {
5444                 flow->tunnel = 1;
5445                 flow->tunnel_id = tunnel->tunnel_id;
5446                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5447                 mlx5_free(default_miss_ctx.queue);
5448         }
5449         mlx5_flow_pop_thread_workspace();
5450         return idx;
5451 error:
5452         MLX5_ASSERT(flow);
5453         ret = rte_errno; /* Save rte_errno before cleanup. */
5454         flow_mreg_del_copy_action(dev, flow);
5455         flow_drv_destroy(dev, flow);
5456         if (rss_desc->shared_rss)
5457                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
5458                         mlx5_ipool_get
5459                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
5460                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
5461         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5462         rte_errno = ret; /* Restore rte_errno. */
5463         ret = rte_errno;
5464         rte_errno = ret;
5465         mlx5_flow_pop_thread_workspace();
5466 error_before_hairpin_split:
5467         rte_free(translated_actions);
5468         return 0;
5469 }
5470
5471 /**
5472  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5473  * incoming packets to table 1.
5474  *
5475  * Other flow rules, requested for group n, will be created in
5476  * e-switch table n+1.
5477  * Jump action to e-switch group n will be created to group n+1.
5478  *
5479  * Used when working in switchdev mode, to utilise advantages of table 1
5480  * and above.
5481  *
5482  * @param dev
5483  *   Pointer to Ethernet device.
5484  *
5485  * @return
5486  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5487  */
5488 struct rte_flow *
5489 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5490 {
5491         const struct rte_flow_attr attr = {
5492                 .group = 0,
5493                 .priority = 0,
5494                 .ingress = 1,
5495                 .egress = 0,
5496                 .transfer = 1,
5497         };
5498         const struct rte_flow_item pattern = {
5499                 .type = RTE_FLOW_ITEM_TYPE_END,
5500         };
5501         struct rte_flow_action_jump jump = {
5502                 .group = 1,
5503         };
5504         const struct rte_flow_action actions[] = {
5505                 {
5506                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5507                         .conf = &jump,
5508                 },
5509                 {
5510                         .type = RTE_FLOW_ACTION_TYPE_END,
5511                 },
5512         };
5513         struct mlx5_priv *priv = dev->data->dev_private;
5514         struct rte_flow_error error;
5515
5516         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5517                                                    &attr, &pattern,
5518                                                    actions, false, &error);
5519 }
5520
5521 /**
5522  * Validate a flow supported by the NIC.
5523  *
5524  * @see rte_flow_validate()
5525  * @see rte_flow_ops
5526  */
5527 int
5528 mlx5_flow_validate(struct rte_eth_dev *dev,
5529                    const struct rte_flow_attr *attr,
5530                    const struct rte_flow_item items[],
5531                    const struct rte_flow_action original_actions[],
5532                    struct rte_flow_error *error)
5533 {
5534         int hairpin_flow;
5535         struct mlx5_translated_shared_action
5536                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5537         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5538         const struct rte_flow_action *actions;
5539         struct rte_flow_action *translated_actions = NULL;
5540         int ret = flow_shared_actions_translate(dev, original_actions,
5541                                                 shared_actions,
5542                                                 &shared_actions_n,
5543                                                 &translated_actions, error);
5544
5545         if (ret)
5546                 return ret;
5547         actions = translated_actions ? translated_actions : original_actions;
5548         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5549         ret = flow_drv_validate(dev, attr, items, actions,
5550                                 true, hairpin_flow, error);
5551         rte_free(translated_actions);
5552         return ret;
5553 }
5554
5555 /**
5556  * Create a flow.
5557  *
5558  * @see rte_flow_create()
5559  * @see rte_flow_ops
5560  */
5561 struct rte_flow *
5562 mlx5_flow_create(struct rte_eth_dev *dev,
5563                  const struct rte_flow_attr *attr,
5564                  const struct rte_flow_item items[],
5565                  const struct rte_flow_action actions[],
5566                  struct rte_flow_error *error)
5567 {
5568         struct mlx5_priv *priv = dev->data->dev_private;
5569
5570         /*
5571          * If the device is not started yet, it is not allowed to created a
5572          * flow from application. PMD default flows and traffic control flows
5573          * are not affected.
5574          */
5575         if (unlikely(!dev->data->dev_started)) {
5576                 DRV_LOG(DEBUG, "port %u is not started when "
5577                         "inserting a flow", dev->data->port_id);
5578                 rte_flow_error_set(error, ENODEV,
5579                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5580                                    NULL,
5581                                    "port not started");
5582                 return NULL;
5583         }
5584
5585         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5586                                   attr, items, actions, true, error);
5587 }
5588
5589 /**
5590  * Destroy a flow in a list.
5591  *
5592  * @param dev
5593  *   Pointer to Ethernet device.
5594  * @param list
5595  *   Pointer to the Indexed flow list. If this parameter NULL,
5596  *   there is no flow removal from the list. Be noted that as
5597  *   flow is add to the indexed list, memory of the indexed
5598  *   list points to maybe changed as flow destroyed.
5599  * @param[in] flow_idx
5600  *   Index of flow to destroy.
5601  */
5602 static void
5603 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5604                   uint32_t flow_idx)
5605 {
5606         struct mlx5_priv *priv = dev->data->dev_private;
5607         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5608                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5609
5610         if (!flow)
5611                 return;
5612         /*
5613          * Update RX queue flags only if port is started, otherwise it is
5614          * already clean.
5615          */
5616         if (dev->data->dev_started)
5617                 flow_rxq_flags_trim(dev, flow);
5618         flow_drv_destroy(dev, flow);
5619         if (list) {
5620                 rte_spinlock_lock(&priv->flow_list_lock);
5621                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5622                              flow_idx, flow, next);
5623                 rte_spinlock_unlock(&priv->flow_list_lock);
5624         }
5625         if (flow->tunnel) {
5626                 struct mlx5_flow_tunnel *tunnel;
5627
5628                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5629                 RTE_VERIFY(tunnel);
5630                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5631                         mlx5_flow_tunnel_free(dev, tunnel);
5632         }
5633         flow_mreg_del_copy_action(dev, flow);
5634         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5635 }
5636
5637 /**
5638  * Destroy all flows.
5639  *
5640  * @param dev
5641  *   Pointer to Ethernet device.
5642  * @param list
5643  *   Pointer to the Indexed flow list.
5644  * @param active
5645  *   If flushing is called avtively.
5646  */
5647 void
5648 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5649 {
5650         uint32_t num_flushed = 0;
5651
5652         while (*list) {
5653                 flow_list_destroy(dev, list, *list);
5654                 num_flushed++;
5655         }
5656         if (active) {
5657                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5658                         dev->data->port_id, num_flushed);
5659         }
5660 }
5661
5662 /**
5663  * Stop all default actions for flows.
5664  *
5665  * @param dev
5666  *   Pointer to Ethernet device.
5667  */
5668 void
5669 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5670 {
5671         flow_mreg_del_default_copy_action(dev);
5672         flow_rxq_flags_clear(dev);
5673 }
5674
5675 /**
5676  * Start all default actions for flows.
5677  *
5678  * @param dev
5679  *   Pointer to Ethernet device.
5680  * @return
5681  *   0 on success, a negative errno value otherwise and rte_errno is set.
5682  */
5683 int
5684 mlx5_flow_start_default(struct rte_eth_dev *dev)
5685 {
5686         struct rte_flow_error error;
5687
5688         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5689         return flow_mreg_add_default_copy_action(dev, &error);
5690 }
5691
5692 /**
5693  * Release key of thread specific flow workspace data.
5694  */
5695 static void
5696 flow_release_workspace(void *data)
5697 {
5698         struct mlx5_flow_workspace *wks = data;
5699         struct mlx5_flow_workspace *next;
5700
5701         while (wks) {
5702                 next = wks->next;
5703                 free(wks->rss_desc.queue);
5704                 free(wks);
5705                 wks = next;
5706         }
5707 }
5708
5709 /**
5710  * Initialize key of thread specific flow workspace data.
5711  */
5712 static void
5713 flow_alloc_workspace(void)
5714 {
5715         if (pthread_key_create(&key_workspace, flow_release_workspace))
5716                 DRV_LOG(ERR, "Can't create flow workspace data thread key.");
5717 }
5718
5719 /**
5720  * Get thread specific current flow workspace.
5721  *
5722  * @return pointer to thread specific flow workspace data, NULL on error.
5723  */
5724 struct mlx5_flow_workspace*
5725 mlx5_flow_get_thread_workspace(void)
5726 {
5727         struct mlx5_flow_workspace *data;
5728
5729         data = pthread_getspecific(key_workspace);
5730         MLX5_ASSERT(data && data->inuse);
5731         if (!data || !data->inuse)
5732                 DRV_LOG(ERR, "flow workspace not initialized.");
5733         return data;
5734 }
5735
5736 /**
5737  * Allocate and init new flow workspace.
5738  *
5739  * @return pointer to flow workspace data, NULL on error.
5740  */
5741 static struct mlx5_flow_workspace*
5742 flow_alloc_thread_workspace(void)
5743 {
5744         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
5745
5746         if (!data) {
5747                 DRV_LOG(ERR, "Failed to allocate flow workspace "
5748                         "memory.");
5749                 return NULL;
5750         }
5751         data->rss_desc.queue = calloc(1,
5752                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
5753         if (!data->rss_desc.queue)
5754                 goto err;
5755         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
5756         return data;
5757 err:
5758         if (data->rss_desc.queue)
5759                 free(data->rss_desc.queue);
5760         free(data);
5761         return NULL;
5762 }
5763
5764 /**
5765  * Get new thread specific flow workspace.
5766  *
5767  * If current workspace inuse, create new one and set as current.
5768  *
5769  * @return pointer to thread specific flow workspace data, NULL on error.
5770  */
5771 static struct mlx5_flow_workspace*
5772 mlx5_flow_push_thread_workspace(void)
5773 {
5774         struct mlx5_flow_workspace *curr;
5775         struct mlx5_flow_workspace *data;
5776
5777         if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
5778                 DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
5779                 return NULL;
5780         }
5781         curr = pthread_getspecific(key_workspace);
5782         if (!curr) {
5783                 data = flow_alloc_thread_workspace();
5784                 if (!data)
5785                         return NULL;
5786         } else if (!curr->inuse) {
5787                 data = curr;
5788         } else if (curr->next) {
5789                 data = curr->next;
5790         } else {
5791                 data = flow_alloc_thread_workspace();
5792                 if (!data)
5793                         return NULL;
5794                 curr->next = data;
5795                 data->prev = curr;
5796         }
5797         data->inuse = 1;
5798         data->flow_idx = 0;
5799         /* Set as current workspace */
5800         if (pthread_setspecific(key_workspace, data))
5801                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5802         return data;
5803 }
5804
5805 /**
5806  * Close current thread specific flow workspace.
5807  *
5808  * If previous workspace available, set it as current.
5809  *
5810  * @return pointer to thread specific flow workspace data, NULL on error.
5811  */
5812 static void
5813 mlx5_flow_pop_thread_workspace(void)
5814 {
5815         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
5816
5817         if (!data)
5818                 return;
5819         if (!data->inuse) {
5820                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
5821                 return;
5822         }
5823         data->inuse = 0;
5824         if (!data->prev)
5825                 return;
5826         if (pthread_setspecific(key_workspace, data->prev))
5827                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5828 }
5829
5830 /**
5831  * Verify the flow list is empty
5832  *
5833  * @param dev
5834  *  Pointer to Ethernet device.
5835  *
5836  * @return the number of flows not released.
5837  */
5838 int
5839 mlx5_flow_verify(struct rte_eth_dev *dev)
5840 {
5841         struct mlx5_priv *priv = dev->data->dev_private;
5842         struct rte_flow *flow;
5843         uint32_t idx;
5844         int ret = 0;
5845
5846         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5847                       flow, next) {
5848                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5849                         dev->data->port_id, (void *)flow);
5850                 ++ret;
5851         }
5852         return ret;
5853 }
5854
5855 /**
5856  * Enable default hairpin egress flow.
5857  *
5858  * @param dev
5859  *   Pointer to Ethernet device.
5860  * @param queue
5861  *   The queue index.
5862  *
5863  * @return
5864  *   0 on success, a negative errno value otherwise and rte_errno is set.
5865  */
5866 int
5867 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5868                             uint32_t queue)
5869 {
5870         struct mlx5_priv *priv = dev->data->dev_private;
5871         const struct rte_flow_attr attr = {
5872                 .egress = 1,
5873                 .priority = 0,
5874         };
5875         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5876                 .queue = queue,
5877         };
5878         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5879                 .queue = UINT32_MAX,
5880         };
5881         struct rte_flow_item items[] = {
5882                 {
5883                         .type = (enum rte_flow_item_type)
5884                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5885                         .spec = &queue_spec,
5886                         .last = NULL,
5887                         .mask = &queue_mask,
5888                 },
5889                 {
5890                         .type = RTE_FLOW_ITEM_TYPE_END,
5891                 },
5892         };
5893         struct rte_flow_action_jump jump = {
5894                 .group = MLX5_HAIRPIN_TX_TABLE,
5895         };
5896         struct rte_flow_action actions[2];
5897         uint32_t flow_idx;
5898         struct rte_flow_error error;
5899
5900         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5901         actions[0].conf = &jump;
5902         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5903         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5904                                 &attr, items, actions, false, &error);
5905         if (!flow_idx) {
5906                 DRV_LOG(DEBUG,
5907                         "Failed to create ctrl flow: rte_errno(%d),"
5908                         " type(%d), message(%s)",
5909                         rte_errno, error.type,
5910                         error.message ? error.message : " (no stated reason)");
5911                 return -rte_errno;
5912         }
5913         return 0;
5914 }
5915
5916 /**
5917  * Enable a control flow configured from the control plane.
5918  *
5919  * @param dev
5920  *   Pointer to Ethernet device.
5921  * @param eth_spec
5922  *   An Ethernet flow spec to apply.
5923  * @param eth_mask
5924  *   An Ethernet flow mask to apply.
5925  * @param vlan_spec
5926  *   A VLAN flow spec to apply.
5927  * @param vlan_mask
5928  *   A VLAN flow mask to apply.
5929  *
5930  * @return
5931  *   0 on success, a negative errno value otherwise and rte_errno is set.
5932  */
5933 int
5934 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5935                     struct rte_flow_item_eth *eth_spec,
5936                     struct rte_flow_item_eth *eth_mask,
5937                     struct rte_flow_item_vlan *vlan_spec,
5938                     struct rte_flow_item_vlan *vlan_mask)
5939 {
5940         struct mlx5_priv *priv = dev->data->dev_private;
5941         const struct rte_flow_attr attr = {
5942                 .ingress = 1,
5943                 .priority = MLX5_FLOW_PRIO_RSVD,
5944         };
5945         struct rte_flow_item items[] = {
5946                 {
5947                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5948                         .spec = eth_spec,
5949                         .last = NULL,
5950                         .mask = eth_mask,
5951                 },
5952                 {
5953                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5954                                               RTE_FLOW_ITEM_TYPE_END,
5955                         .spec = vlan_spec,
5956                         .last = NULL,
5957                         .mask = vlan_mask,
5958                 },
5959                 {
5960                         .type = RTE_FLOW_ITEM_TYPE_END,
5961                 },
5962         };
5963         uint16_t queue[priv->reta_idx_n];
5964         struct rte_flow_action_rss action_rss = {
5965                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5966                 .level = 0,
5967                 .types = priv->rss_conf.rss_hf,
5968                 .key_len = priv->rss_conf.rss_key_len,
5969                 .queue_num = priv->reta_idx_n,
5970                 .key = priv->rss_conf.rss_key,
5971                 .queue = queue,
5972         };
5973         struct rte_flow_action actions[] = {
5974                 {
5975                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5976                         .conf = &action_rss,
5977                 },
5978                 {
5979                         .type = RTE_FLOW_ACTION_TYPE_END,
5980                 },
5981         };
5982         uint32_t flow_idx;
5983         struct rte_flow_error error;
5984         unsigned int i;
5985
5986         if (!priv->reta_idx_n || !priv->rxqs_n) {
5987                 return 0;
5988         }
5989         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5990                 action_rss.types = 0;
5991         for (i = 0; i != priv->reta_idx_n; ++i)
5992                 queue[i] = (*priv->reta_idx)[i];
5993         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5994                                 &attr, items, actions, false, &error);
5995         if (!flow_idx)
5996                 return -rte_errno;
5997         return 0;
5998 }
5999
6000 /**
6001  * Enable a flow control configured from the control plane.
6002  *
6003  * @param dev
6004  *   Pointer to Ethernet device.
6005  * @param eth_spec
6006  *   An Ethernet flow spec to apply.
6007  * @param eth_mask
6008  *   An Ethernet flow mask to apply.
6009  *
6010  * @return
6011  *   0 on success, a negative errno value otherwise and rte_errno is set.
6012  */
6013 int
6014 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6015                struct rte_flow_item_eth *eth_spec,
6016                struct rte_flow_item_eth *eth_mask)
6017 {
6018         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6019 }
6020
6021 /**
6022  * Create default miss flow rule matching lacp traffic
6023  *
6024  * @param dev
6025  *   Pointer to Ethernet device.
6026  * @param eth_spec
6027  *   An Ethernet flow spec to apply.
6028  *
6029  * @return
6030  *   0 on success, a negative errno value otherwise and rte_errno is set.
6031  */
6032 int
6033 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6034 {
6035         struct mlx5_priv *priv = dev->data->dev_private;
6036         /*
6037          * The LACP matching is done by only using ether type since using
6038          * a multicast dst mac causes kernel to give low priority to this flow.
6039          */
6040         static const struct rte_flow_item_eth lacp_spec = {
6041                 .type = RTE_BE16(0x8809),
6042         };
6043         static const struct rte_flow_item_eth lacp_mask = {
6044                 .type = 0xffff,
6045         };
6046         const struct rte_flow_attr attr = {
6047                 .ingress = 1,
6048         };
6049         struct rte_flow_item items[] = {
6050                 {
6051                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6052                         .spec = &lacp_spec,
6053                         .mask = &lacp_mask,
6054                 },
6055                 {
6056                         .type = RTE_FLOW_ITEM_TYPE_END,
6057                 },
6058         };
6059         struct rte_flow_action actions[] = {
6060                 {
6061                         .type = (enum rte_flow_action_type)
6062                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6063                 },
6064                 {
6065                         .type = RTE_FLOW_ACTION_TYPE_END,
6066                 },
6067         };
6068         struct rte_flow_error error;
6069         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6070                                 &attr, items, actions, false, &error);
6071
6072         if (!flow_idx)
6073                 return -rte_errno;
6074         return 0;
6075 }
6076
6077 /**
6078  * Destroy a flow.
6079  *
6080  * @see rte_flow_destroy()
6081  * @see rte_flow_ops
6082  */
6083 int
6084 mlx5_flow_destroy(struct rte_eth_dev *dev,
6085                   struct rte_flow *flow,
6086                   struct rte_flow_error *error __rte_unused)
6087 {
6088         struct mlx5_priv *priv = dev->data->dev_private;
6089
6090         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6091         return 0;
6092 }
6093
6094 /**
6095  * Destroy all flows.
6096  *
6097  * @see rte_flow_flush()
6098  * @see rte_flow_ops
6099  */
6100 int
6101 mlx5_flow_flush(struct rte_eth_dev *dev,
6102                 struct rte_flow_error *error __rte_unused)
6103 {
6104         struct mlx5_priv *priv = dev->data->dev_private;
6105
6106         mlx5_flow_list_flush(dev, &priv->flows, false);
6107         return 0;
6108 }
6109
6110 /**
6111  * Isolated mode.
6112  *
6113  * @see rte_flow_isolate()
6114  * @see rte_flow_ops
6115  */
6116 int
6117 mlx5_flow_isolate(struct rte_eth_dev *dev,
6118                   int enable,
6119                   struct rte_flow_error *error)
6120 {
6121         struct mlx5_priv *priv = dev->data->dev_private;
6122
6123         if (dev->data->dev_started) {
6124                 rte_flow_error_set(error, EBUSY,
6125                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6126                                    NULL,
6127                                    "port must be stopped first");
6128                 return -rte_errno;
6129         }
6130         priv->isolated = !!enable;
6131         if (enable)
6132                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6133         else
6134                 dev->dev_ops = &mlx5_os_dev_ops;
6135
6136         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6137         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6138
6139         return 0;
6140 }
6141
6142 /**
6143  * Query a flow.
6144  *
6145  * @see rte_flow_query()
6146  * @see rte_flow_ops
6147  */
6148 static int
6149 flow_drv_query(struct rte_eth_dev *dev,
6150                uint32_t flow_idx,
6151                const struct rte_flow_action *actions,
6152                void *data,
6153                struct rte_flow_error *error)
6154 {
6155         struct mlx5_priv *priv = dev->data->dev_private;
6156         const struct mlx5_flow_driver_ops *fops;
6157         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6158                                                [MLX5_IPOOL_RTE_FLOW],
6159                                                flow_idx);
6160         enum mlx5_flow_drv_type ftype;
6161
6162         if (!flow) {
6163                 return rte_flow_error_set(error, ENOENT,
6164                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6165                           NULL,
6166                           "invalid flow handle");
6167         }
6168         ftype = flow->drv_type;
6169         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6170         fops = flow_get_drv_ops(ftype);
6171
6172         return fops->query(dev, flow, actions, data, error);
6173 }
6174
6175 /**
6176  * Query a flow.
6177  *
6178  * @see rte_flow_query()
6179  * @see rte_flow_ops
6180  */
6181 int
6182 mlx5_flow_query(struct rte_eth_dev *dev,
6183                 struct rte_flow *flow,
6184                 const struct rte_flow_action *actions,
6185                 void *data,
6186                 struct rte_flow_error *error)
6187 {
6188         int ret;
6189
6190         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6191                              error);
6192         if (ret < 0)
6193                 return ret;
6194         return 0;
6195 }
6196
6197 /**
6198  * Manage filter operations.
6199  *
6200  * @param dev
6201  *   Pointer to Ethernet device structure.
6202  * @param filter_type
6203  *   Filter type.
6204  * @param filter_op
6205  *   Operation to perform.
6206  * @param arg
6207  *   Pointer to operation-specific structure.
6208  *
6209  * @return
6210  *   0 on success, a negative errno value otherwise and rte_errno is set.
6211  */
6212 int
6213 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6214                      enum rte_filter_type filter_type,
6215                      enum rte_filter_op filter_op,
6216                      void *arg)
6217 {
6218         switch (filter_type) {
6219         case RTE_ETH_FILTER_GENERIC:
6220                 if (filter_op != RTE_ETH_FILTER_GET) {
6221                         rte_errno = EINVAL;
6222                         return -rte_errno;
6223                 }
6224                 *(const void **)arg = &mlx5_flow_ops;
6225                 return 0;
6226         default:
6227                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6228                         dev->data->port_id, filter_type);
6229                 rte_errno = ENOTSUP;
6230                 return -rte_errno;
6231         }
6232         return 0;
6233 }
6234
6235 /**
6236  * Create the needed meter and suffix tables.
6237  *
6238  * @param[in] dev
6239  *   Pointer to Ethernet device.
6240  * @param[in] fm
6241  *   Pointer to the flow meter.
6242  *
6243  * @return
6244  *   Pointer to table set on success, NULL otherwise.
6245  */
6246 struct mlx5_meter_domains_infos *
6247 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6248                           const struct mlx5_flow_meter *fm)
6249 {
6250         const struct mlx5_flow_driver_ops *fops;
6251
6252         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6253         return fops->create_mtr_tbls(dev, fm);
6254 }
6255
6256 /**
6257  * Destroy the meter table set.
6258  *
6259  * @param[in] dev
6260  *   Pointer to Ethernet device.
6261  * @param[in] tbl
6262  *   Pointer to the meter table set.
6263  *
6264  * @return
6265  *   0 on success.
6266  */
6267 int
6268 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6269                            struct mlx5_meter_domains_infos *tbls)
6270 {
6271         const struct mlx5_flow_driver_ops *fops;
6272
6273         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6274         return fops->destroy_mtr_tbls(dev, tbls);
6275 }
6276
6277 /**
6278  * Create policer rules.
6279  *
6280  * @param[in] dev
6281  *   Pointer to Ethernet device.
6282  * @param[in] fm
6283  *   Pointer to flow meter structure.
6284  * @param[in] attr
6285  *   Pointer to flow attributes.
6286  *
6287  * @return
6288  *   0 on success, -1 otherwise.
6289  */
6290 int
6291 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6292                                struct mlx5_flow_meter *fm,
6293                                const struct rte_flow_attr *attr)
6294 {
6295         const struct mlx5_flow_driver_ops *fops;
6296
6297         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6298         return fops->create_policer_rules(dev, fm, attr);
6299 }
6300
6301 /**
6302  * Destroy policer rules.
6303  *
6304  * @param[in] fm
6305  *   Pointer to flow meter structure.
6306  * @param[in] attr
6307  *   Pointer to flow attributes.
6308  *
6309  * @return
6310  *   0 on success, -1 otherwise.
6311  */
6312 int
6313 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6314                                 struct mlx5_flow_meter *fm,
6315                                 const struct rte_flow_attr *attr)
6316 {
6317         const struct mlx5_flow_driver_ops *fops;
6318
6319         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6320         return fops->destroy_policer_rules(dev, fm, attr);
6321 }
6322
6323 /**
6324  * Allocate a counter.
6325  *
6326  * @param[in] dev
6327  *   Pointer to Ethernet device structure.
6328  *
6329  * @return
6330  *   Index to allocated counter  on success, 0 otherwise.
6331  */
6332 uint32_t
6333 mlx5_counter_alloc(struct rte_eth_dev *dev)
6334 {
6335         const struct mlx5_flow_driver_ops *fops;
6336         struct rte_flow_attr attr = { .transfer = 0 };
6337
6338         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6339                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6340                 return fops->counter_alloc(dev);
6341         }
6342         DRV_LOG(ERR,
6343                 "port %u counter allocate is not supported.",
6344                  dev->data->port_id);
6345         return 0;
6346 }
6347
6348 /**
6349  * Free a counter.
6350  *
6351  * @param[in] dev
6352  *   Pointer to Ethernet device structure.
6353  * @param[in] cnt
6354  *   Index to counter to be free.
6355  */
6356 void
6357 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6358 {
6359         const struct mlx5_flow_driver_ops *fops;
6360         struct rte_flow_attr attr = { .transfer = 0 };
6361
6362         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6363                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6364                 fops->counter_free(dev, cnt);
6365                 return;
6366         }
6367         DRV_LOG(ERR,
6368                 "port %u counter free is not supported.",
6369                  dev->data->port_id);
6370 }
6371
6372 /**
6373  * Query counter statistics.
6374  *
6375  * @param[in] dev
6376  *   Pointer to Ethernet device structure.
6377  * @param[in] cnt
6378  *   Index to counter to query.
6379  * @param[in] clear
6380  *   Set to clear counter statistics.
6381  * @param[out] pkts
6382  *   The counter hits packets number to save.
6383  * @param[out] bytes
6384  *   The counter hits bytes number to save.
6385  *
6386  * @return
6387  *   0 on success, a negative errno value otherwise.
6388  */
6389 int
6390 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6391                    bool clear, uint64_t *pkts, uint64_t *bytes)
6392 {
6393         const struct mlx5_flow_driver_ops *fops;
6394         struct rte_flow_attr attr = { .transfer = 0 };
6395
6396         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6397                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6398                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6399         }
6400         DRV_LOG(ERR,
6401                 "port %u counter query is not supported.",
6402                  dev->data->port_id);
6403         return -ENOTSUP;
6404 }
6405
6406 /**
6407  * Allocate a new memory for the counter values wrapped by all the needed
6408  * management.
6409  *
6410  * @param[in] sh
6411  *   Pointer to mlx5_dev_ctx_shared object.
6412  *
6413  * @return
6414  *   0 on success, a negative errno value otherwise.
6415  */
6416 static int
6417 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
6418 {
6419         struct mlx5_devx_mkey_attr mkey_attr;
6420         struct mlx5_counter_stats_mem_mng *mem_mng;
6421         volatile struct flow_counter_stats *raw_data;
6422         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
6423         int size = (sizeof(struct flow_counter_stats) *
6424                         MLX5_COUNTERS_PER_POOL +
6425                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
6426                         sizeof(struct mlx5_counter_stats_mem_mng);
6427         size_t pgsize = rte_mem_page_size();
6428         uint8_t *mem;
6429         int i;
6430
6431         if (pgsize == (size_t)-1) {
6432                 DRV_LOG(ERR, "Failed to get mem page size");
6433                 rte_errno = ENOMEM;
6434                 return -ENOMEM;
6435         }
6436         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
6437         if (!mem) {
6438                 rte_errno = ENOMEM;
6439                 return -ENOMEM;
6440         }
6441         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
6442         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
6443         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
6444                                                  IBV_ACCESS_LOCAL_WRITE);
6445         if (!mem_mng->umem) {
6446                 rte_errno = errno;
6447                 mlx5_free(mem);
6448                 return -rte_errno;
6449         }
6450         mkey_attr.addr = (uintptr_t)mem;
6451         mkey_attr.size = size;
6452         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
6453         mkey_attr.pd = sh->pdn;
6454         mkey_attr.log_entity_size = 0;
6455         mkey_attr.pg_access = 0;
6456         mkey_attr.klm_array = NULL;
6457         mkey_attr.klm_num = 0;
6458         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
6459         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
6460         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
6461         if (!mem_mng->dm) {
6462                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
6463                 rte_errno = errno;
6464                 mlx5_free(mem);
6465                 return -rte_errno;
6466         }
6467         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
6468         raw_data = (volatile struct flow_counter_stats *)mem;
6469         for (i = 0; i < raws_n; ++i) {
6470                 mem_mng->raws[i].mem_mng = mem_mng;
6471                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
6472         }
6473         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
6474                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
6475                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
6476                                  next);
6477         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
6478         sh->cmng.mem_mng = mem_mng;
6479         return 0;
6480 }
6481
6482 /**
6483  * Set the statistic memory to the new counter pool.
6484  *
6485  * @param[in] sh
6486  *   Pointer to mlx5_dev_ctx_shared object.
6487  * @param[in] pool
6488  *   Pointer to the pool to set the statistic memory.
6489  *
6490  * @return
6491  *   0 on success, a negative errno value otherwise.
6492  */
6493 static int
6494 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
6495                                struct mlx5_flow_counter_pool *pool)
6496 {
6497         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6498         /* Resize statistic memory once used out. */
6499         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
6500             mlx5_flow_create_counter_stat_mem_mng(sh)) {
6501                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
6502                 return -1;
6503         }
6504         rte_spinlock_lock(&pool->sl);
6505         pool->raw = cmng->mem_mng->raws + pool->index %
6506                     MLX5_CNT_CONTAINER_RESIZE;
6507         rte_spinlock_unlock(&pool->sl);
6508         pool->raw_hw = NULL;
6509         return 0;
6510 }
6511
6512 #define MLX5_POOL_QUERY_FREQ_US 1000000
6513
6514 /**
6515  * Set the periodic procedure for triggering asynchronous batch queries for all
6516  * the counter pools.
6517  *
6518  * @param[in] sh
6519  *   Pointer to mlx5_dev_ctx_shared object.
6520  */
6521 void
6522 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6523 {
6524         uint32_t pools_n, us;
6525
6526         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
6527         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6528         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6529         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6530                 sh->cmng.query_thread_on = 0;
6531                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6532         } else {
6533                 sh->cmng.query_thread_on = 1;
6534         }
6535 }
6536
6537 /**
6538  * The periodic procedure for triggering asynchronous batch queries for all the
6539  * counter pools. This function is probably called by the host thread.
6540  *
6541  * @param[in] arg
6542  *   The parameter for the alarm process.
6543  */
6544 void
6545 mlx5_flow_query_alarm(void *arg)
6546 {
6547         struct mlx5_dev_ctx_shared *sh = arg;
6548         int ret;
6549         uint16_t pool_index = sh->cmng.pool_index;
6550         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6551         struct mlx5_flow_counter_pool *pool;
6552         uint16_t n_valid;
6553
6554         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6555                 goto set_alarm;
6556         rte_spinlock_lock(&cmng->pool_update_sl);
6557         pool = cmng->pools[pool_index];
6558         n_valid = cmng->n_valid;
6559         rte_spinlock_unlock(&cmng->pool_update_sl);
6560         /* Set the statistic memory to the new created pool. */
6561         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
6562                 goto set_alarm;
6563         if (pool->raw_hw)
6564                 /* There is a pool query in progress. */
6565                 goto set_alarm;
6566         pool->raw_hw =
6567                 LIST_FIRST(&sh->cmng.free_stat_raws);
6568         if (!pool->raw_hw)
6569                 /* No free counter statistics raw memory. */
6570                 goto set_alarm;
6571         /*
6572          * Identify the counters released between query trigger and query
6573          * handle more efficiently. The counter released in this gap period
6574          * should wait for a new round of query as the new arrived packets
6575          * will not be taken into account.
6576          */
6577         pool->query_gen++;
6578         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
6579                                                MLX5_COUNTERS_PER_POOL,
6580                                                NULL, NULL,
6581                                                pool->raw_hw->mem_mng->dm->id,
6582                                                (void *)(uintptr_t)
6583                                                pool->raw_hw->data,
6584                                                sh->devx_comp,
6585                                                (uint64_t)(uintptr_t)pool);
6586         if (ret) {
6587                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6588                         " %d", pool->min_dcs->id);
6589                 pool->raw_hw = NULL;
6590                 goto set_alarm;
6591         }
6592         LIST_REMOVE(pool->raw_hw, next);
6593         sh->cmng.pending_queries++;
6594         pool_index++;
6595         if (pool_index >= n_valid)
6596                 pool_index = 0;
6597 set_alarm:
6598         sh->cmng.pool_index = pool_index;
6599         mlx5_set_query_alarm(sh);
6600 }
6601
6602 /**
6603  * Check and callback event for new aged flow in the counter pool
6604  *
6605  * @param[in] sh
6606  *   Pointer to mlx5_dev_ctx_shared object.
6607  * @param[in] pool
6608  *   Pointer to Current counter pool.
6609  */
6610 static void
6611 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6612                    struct mlx5_flow_counter_pool *pool)
6613 {
6614         struct mlx5_priv *priv;
6615         struct mlx5_flow_counter *cnt;
6616         struct mlx5_age_info *age_info;
6617         struct mlx5_age_param *age_param;
6618         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6619         struct mlx5_counter_stats_raw *prev = pool->raw;
6620         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
6621         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
6622         uint16_t expected = AGE_CANDIDATE;
6623         uint32_t i;
6624
6625         pool->time_of_last_age_check = curr_time;
6626         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6627                 cnt = MLX5_POOL_GET_CNT(pool, i);
6628                 age_param = MLX5_CNT_TO_AGE(cnt);
6629                 if (__atomic_load_n(&age_param->state,
6630                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
6631                         continue;
6632                 if (cur->data[i].hits != prev->data[i].hits) {
6633                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
6634                                          __ATOMIC_RELAXED);
6635                         continue;
6636                 }
6637                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
6638                                        time_delta,
6639                                        __ATOMIC_RELAXED) <= age_param->timeout)
6640                         continue;
6641                 /**
6642                  * Hold the lock first, or if between the
6643                  * state AGE_TMOUT and tailq operation the
6644                  * release happened, the release procedure
6645                  * may delete a non-existent tailq node.
6646                  */
6647                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6648                 age_info = GET_PORT_AGE_INFO(priv);
6649                 rte_spinlock_lock(&age_info->aged_sl);
6650                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
6651                                                 AGE_TMOUT, false,
6652                                                 __ATOMIC_RELAXED,
6653                                                 __ATOMIC_RELAXED)) {
6654                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6655                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6656                 }
6657                 rte_spinlock_unlock(&age_info->aged_sl);
6658         }
6659         mlx5_age_event_prepare(sh);
6660 }
6661
6662 /**
6663  * Handler for the HW respond about ready values from an asynchronous batch
6664  * query. This function is probably called by the host thread.
6665  *
6666  * @param[in] sh
6667  *   The pointer to the shared device context.
6668  * @param[in] async_id
6669  *   The Devx async ID.
6670  * @param[in] status
6671  *   The status of the completion.
6672  */
6673 void
6674 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6675                                   uint64_t async_id, int status)
6676 {
6677         struct mlx5_flow_counter_pool *pool =
6678                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6679         struct mlx5_counter_stats_raw *raw_to_free;
6680         uint8_t query_gen = pool->query_gen ^ 1;
6681         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6682         enum mlx5_counter_type cnt_type =
6683                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6684                                 MLX5_COUNTER_TYPE_ORIGIN;
6685
6686         if (unlikely(status)) {
6687                 raw_to_free = pool->raw_hw;
6688         } else {
6689                 raw_to_free = pool->raw;
6690                 if (pool->is_aged)
6691                         mlx5_flow_aging_check(sh, pool);
6692                 rte_spinlock_lock(&pool->sl);
6693                 pool->raw = pool->raw_hw;
6694                 rte_spinlock_unlock(&pool->sl);
6695                 /* Be sure the new raw counters data is updated in memory. */
6696                 rte_io_wmb();
6697                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6698                         rte_spinlock_lock(&cmng->csl[cnt_type]);
6699                         TAILQ_CONCAT(&cmng->counters[cnt_type],
6700                                      &pool->counters[query_gen], next);
6701                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6702                 }
6703         }
6704         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6705         pool->raw_hw = NULL;
6706         sh->cmng.pending_queries--;
6707 }
6708
6709 static int
6710 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
6711                     const struct flow_grp_info *grp_info,
6712                     struct rte_flow_error *error)
6713 {
6714         if (grp_info->transfer && grp_info->external &&
6715             grp_info->fdb_def_rule) {
6716                 if (group == UINT32_MAX)
6717                         return rte_flow_error_set
6718                                                 (error, EINVAL,
6719                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6720                                                  NULL,
6721                                                  "group index not supported");
6722                 *table = group + 1;
6723         } else {
6724                 *table = group;
6725         }
6726         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
6727         return 0;
6728 }
6729
6730 /**
6731  * Translate the rte_flow group index to HW table value.
6732  *
6733  * If tunnel offload is disabled, all group ids converted to flow table
6734  * id using the standard method.
6735  * If tunnel offload is enabled, group id can be converted using the
6736  * standard or tunnel conversion method. Group conversion method
6737  * selection depends on flags in `grp_info` parameter:
6738  * - Internal (grp_info.external == 0) groups conversion uses the
6739  *   standard method.
6740  * - Group ids in JUMP action converted with the tunnel conversion.
6741  * - Group id in rule attribute conversion depends on a rule type and
6742  *   group id value:
6743  *   ** non zero group attributes converted with the tunnel method
6744  *   ** zero group attribute in non-tunnel rule is converted using the
6745  *      standard method - there's only one root table
6746  *   ** zero group attribute in steer tunnel rule is converted with the
6747  *      standard method - single root table
6748  *   ** zero group attribute in match tunnel rule is a special OvS
6749  *      case: that value is used for portability reasons. That group
6750  *      id is converted with the tunnel conversion method.
6751  *
6752  * @param[in] dev
6753  *   Port device
6754  * @param[in] tunnel
6755  *   PMD tunnel offload object
6756  * @param[in] group
6757  *   rte_flow group index value.
6758  * @param[out] table
6759  *   HW table value.
6760  * @param[in] grp_info
6761  *   flags used for conversion
6762  * @param[out] error
6763  *   Pointer to error structure.
6764  *
6765  * @return
6766  *   0 on success, a negative errno value otherwise and rte_errno is set.
6767  */
6768 int
6769 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
6770                          const struct mlx5_flow_tunnel *tunnel,
6771                          uint32_t group, uint32_t *table,
6772                          const struct flow_grp_info *grp_info,
6773                          struct rte_flow_error *error)
6774 {
6775         int ret;
6776         bool standard_translation;
6777
6778         if (!grp_info->skip_scale && grp_info->external &&
6779             group < MLX5_MAX_TABLES_EXTERNAL)
6780                 group *= MLX5_FLOW_TABLE_FACTOR;
6781         if (is_tunnel_offload_active(dev)) {
6782                 standard_translation = !grp_info->external ||
6783                                         grp_info->std_tbl_fix;
6784         } else {
6785                 standard_translation = true;
6786         }
6787         DRV_LOG(DEBUG,
6788                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
6789                 dev->data->port_id, group, grp_info->transfer,
6790                 grp_info->external, grp_info->fdb_def_rule,
6791                 standard_translation ? "STANDARD" : "TUNNEL");
6792         if (standard_translation)
6793                 ret = flow_group_to_table(dev->data->port_id, group, table,
6794                                           grp_info, error);
6795         else
6796                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
6797                                                       table, error);
6798
6799         return ret;
6800 }
6801
6802 /**
6803  * Discover availability of metadata reg_c's.
6804  *
6805  * Iteratively use test flows to check availability.
6806  *
6807  * @param[in] dev
6808  *   Pointer to the Ethernet device structure.
6809  *
6810  * @return
6811  *   0 on success, a negative errno value otherwise and rte_errno is set.
6812  */
6813 int
6814 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6815 {
6816         struct mlx5_priv *priv = dev->data->dev_private;
6817         struct mlx5_dev_config *config = &priv->config;
6818         enum modify_reg idx;
6819         int n = 0;
6820
6821         /* reg_c[0] and reg_c[1] are reserved. */
6822         config->flow_mreg_c[n++] = REG_C_0;
6823         config->flow_mreg_c[n++] = REG_C_1;
6824         /* Discover availability of other reg_c's. */
6825         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6826                 struct rte_flow_attr attr = {
6827                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6828                         .priority = MLX5_FLOW_PRIO_RSVD,
6829                         .ingress = 1,
6830                 };
6831                 struct rte_flow_item items[] = {
6832                         [0] = {
6833                                 .type = RTE_FLOW_ITEM_TYPE_END,
6834                         },
6835                 };
6836                 struct rte_flow_action actions[] = {
6837                         [0] = {
6838                                 .type = (enum rte_flow_action_type)
6839                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6840                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6841                                         .src = REG_C_1,
6842                                         .dst = idx,
6843                                 },
6844                         },
6845                         [1] = {
6846                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6847                                 .conf = &(struct rte_flow_action_jump){
6848                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6849                                 },
6850                         },
6851                         [2] = {
6852                                 .type = RTE_FLOW_ACTION_TYPE_END,
6853                         },
6854                 };
6855                 uint32_t flow_idx;
6856                 struct rte_flow *flow;
6857                 struct rte_flow_error error;
6858
6859                 if (!config->dv_flow_en)
6860                         break;
6861                 /* Create internal flow, validation skips copy action. */
6862                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6863                                             actions, false, &error);
6864                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6865                                       flow_idx);
6866                 if (!flow)
6867                         continue;
6868                 config->flow_mreg_c[n++] = idx;
6869                 flow_list_destroy(dev, NULL, flow_idx);
6870         }
6871         for (; n < MLX5_MREG_C_NUM; ++n)
6872                 config->flow_mreg_c[n] = REG_NON;
6873         return 0;
6874 }
6875
6876 /**
6877  * Dump flow raw hw data to file
6878  *
6879  * @param[in] dev
6880  *    The pointer to Ethernet device.
6881  * @param[in] file
6882  *   A pointer to a file for output.
6883  * @param[out] error
6884  *   Perform verbose error reporting if not NULL. PMDs initialize this
6885  *   structure in case of error only.
6886  * @return
6887  *   0 on success, a nagative value otherwise.
6888  */
6889 int
6890 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6891                    FILE *file,
6892                    struct rte_flow_error *error __rte_unused)
6893 {
6894         struct mlx5_priv *priv = dev->data->dev_private;
6895         struct mlx5_dev_ctx_shared *sh = priv->sh;
6896
6897         if (!priv->config.dv_flow_en) {
6898                 if (fputs("device dv flow disabled\n", file) <= 0)
6899                         return -errno;
6900                 return -ENOTSUP;
6901         }
6902         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6903                                        sh->tx_domain, file);
6904 }
6905
6906 /**
6907  * Get aged-out flows.
6908  *
6909  * @param[in] dev
6910  *   Pointer to the Ethernet device structure.
6911  * @param[in] context
6912  *   The address of an array of pointers to the aged-out flows contexts.
6913  * @param[in] nb_countexts
6914  *   The length of context array pointers.
6915  * @param[out] error
6916  *   Perform verbose error reporting if not NULL. Initialized in case of
6917  *   error only.
6918  *
6919  * @return
6920  *   how many contexts get in success, otherwise negative errno value.
6921  *   if nb_contexts is 0, return the amount of all aged contexts.
6922  *   if nb_contexts is not 0 , return the amount of aged flows reported
6923  *   in the context array.
6924  */
6925 int
6926 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6927                         uint32_t nb_contexts, struct rte_flow_error *error)
6928 {
6929         const struct mlx5_flow_driver_ops *fops;
6930         struct rte_flow_attr attr = { .transfer = 0 };
6931
6932         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6933                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6934                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6935                                                     error);
6936         }
6937         DRV_LOG(ERR,
6938                 "port %u get aged flows is not supported.",
6939                  dev->data->port_id);
6940         return -ENOTSUP;
6941 }
6942
6943 /* Wrapper for driver action_validate op callback */
6944 static int
6945 flow_drv_action_validate(struct rte_eth_dev *dev,
6946                          const struct rte_flow_shared_action_conf *conf,
6947                          const struct rte_flow_action *action,
6948                          const struct mlx5_flow_driver_ops *fops,
6949                          struct rte_flow_error *error)
6950 {
6951         static const char err_msg[] = "shared action validation unsupported";
6952
6953         if (!fops->action_validate) {
6954                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6955                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6956                                    NULL, err_msg);
6957                 return -rte_errno;
6958         }
6959         return fops->action_validate(dev, conf, action, error);
6960 }
6961
6962 /**
6963  * Destroys the shared action by handle.
6964  *
6965  * @param dev
6966  *   Pointer to Ethernet device structure.
6967  * @param[in] action
6968  *   Handle for the shared action to be destroyed.
6969  * @param[out] error
6970  *   Perform verbose error reporting if not NULL. PMDs initialize this
6971  *   structure in case of error only.
6972  *
6973  * @return
6974  *   0 on success, a negative errno value otherwise and rte_errno is set.
6975  *
6976  * @note: wrapper for driver action_create op callback.
6977  */
6978 static int
6979 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
6980                            struct rte_flow_shared_action *action,
6981                            struct rte_flow_error *error)
6982 {
6983         static const char err_msg[] = "shared action destruction unsupported";
6984         struct rte_flow_attr attr = { .transfer = 0 };
6985         const struct mlx5_flow_driver_ops *fops =
6986                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
6987
6988         if (!fops->action_destroy) {
6989                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6990                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6991                                    NULL, err_msg);
6992                 return -rte_errno;
6993         }
6994         return fops->action_destroy(dev, action, error);
6995 }
6996
6997 /* Wrapper for driver action_destroy op callback */
6998 static int
6999 flow_drv_action_update(struct rte_eth_dev *dev,
7000                        struct rte_flow_shared_action *action,
7001                        const void *action_conf,
7002                        const struct mlx5_flow_driver_ops *fops,
7003                        struct rte_flow_error *error)
7004 {
7005         static const char err_msg[] = "shared action update unsupported";
7006
7007         if (!fops->action_update) {
7008                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7009                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7010                                    NULL, err_msg);
7011                 return -rte_errno;
7012         }
7013         return fops->action_update(dev, action, action_conf, error);
7014 }
7015
7016 /* Wrapper for driver action_destroy op callback */
7017 static int
7018 flow_drv_action_query(struct rte_eth_dev *dev,
7019                       const struct rte_flow_shared_action *action,
7020                       void *data,
7021                       const struct mlx5_flow_driver_ops *fops,
7022                       struct rte_flow_error *error)
7023 {
7024         static const char err_msg[] = "shared action query unsupported";
7025
7026         if (!fops->action_query) {
7027                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7028                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7029                                    NULL, err_msg);
7030                 return -rte_errno;
7031         }
7032         return fops->action_query(dev, action, data, error);
7033 }
7034
7035 /**
7036  * Create shared action for reuse in multiple flow rules.
7037  *
7038  * @param dev
7039  *   Pointer to Ethernet device structure.
7040  * @param[in] action
7041  *   Action configuration for shared action creation.
7042  * @param[out] error
7043  *   Perform verbose error reporting if not NULL. PMDs initialize this
7044  *   structure in case of error only.
7045  * @return
7046  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7047  */
7048 static struct rte_flow_shared_action *
7049 mlx5_shared_action_create(struct rte_eth_dev *dev,
7050                           const struct rte_flow_shared_action_conf *conf,
7051                           const struct rte_flow_action *action,
7052                           struct rte_flow_error *error)
7053 {
7054         static const char err_msg[] = "shared action creation unsupported";
7055         struct rte_flow_attr attr = { .transfer = 0 };
7056         const struct mlx5_flow_driver_ops *fops =
7057                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7058
7059         if (flow_drv_action_validate(dev, conf, action, fops, error))
7060                 return NULL;
7061         if (!fops->action_create) {
7062                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7063                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7064                                    NULL, err_msg);
7065                 return NULL;
7066         }
7067         return fops->action_create(dev, conf, action, error);
7068 }
7069
7070 /**
7071  * Updates inplace the shared action configuration pointed by *action* handle
7072  * with the configuration provided as *action* argument.
7073  * The update of the shared action configuration effects all flow rules reusing
7074  * the action via handle.
7075  *
7076  * @param dev
7077  *   Pointer to Ethernet device structure.
7078  * @param[in] shared_action
7079  *   Handle for the shared action to be updated.
7080  * @param[in] action
7081  *   Action specification used to modify the action pointed by handle.
7082  *   *action* should be of same type with the action pointed by the *action*
7083  *   handle argument, otherwise considered as invalid.
7084  * @param[out] error
7085  *   Perform verbose error reporting if not NULL. PMDs initialize this
7086  *   structure in case of error only.
7087  *
7088  * @return
7089  *   0 on success, a negative errno value otherwise and rte_errno is set.
7090  */
7091 static int
7092 mlx5_shared_action_update(struct rte_eth_dev *dev,
7093                 struct rte_flow_shared_action *shared_action,
7094                 const struct rte_flow_action *action,
7095                 struct rte_flow_error *error)
7096 {
7097         struct rte_flow_attr attr = { .transfer = 0 };
7098         const struct mlx5_flow_driver_ops *fops =
7099                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7100         int ret;
7101
7102         ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7103         if (ret)
7104                 return ret;
7105         return flow_drv_action_update(dev, shared_action, action->conf, fops,
7106                                       error);
7107 }
7108
7109 /**
7110  * Query the shared action by handle.
7111  *
7112  * This function allows retrieving action-specific data such as counters.
7113  * Data is gathered by special action which may be present/referenced in
7114  * more than one flow rule definition.
7115  *
7116  * \see RTE_FLOW_ACTION_TYPE_COUNT
7117  *
7118  * @param dev
7119  *   Pointer to Ethernet device structure.
7120  * @param[in] action
7121  *   Handle for the shared action to query.
7122  * @param[in, out] data
7123  *   Pointer to storage for the associated query data type.
7124  * @param[out] error
7125  *   Perform verbose error reporting if not NULL. PMDs initialize this
7126  *   structure in case of error only.
7127  *
7128  * @return
7129  *   0 on success, a negative errno value otherwise and rte_errno is set.
7130  */
7131 static int
7132 mlx5_shared_action_query(struct rte_eth_dev *dev,
7133                          const struct rte_flow_shared_action *action,
7134                          void *data,
7135                          struct rte_flow_error *error)
7136 {
7137         struct rte_flow_attr attr = { .transfer = 0 };
7138         const struct mlx5_flow_driver_ops *fops =
7139                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7140
7141         return flow_drv_action_query(dev, action, data, fops, error);
7142 }
7143
7144 /**
7145  * Destroy all shared actions.
7146  *
7147  * @param dev
7148  *   Pointer to Ethernet device.
7149  *
7150  * @return
7151  *   0 on success, a negative errno value otherwise and rte_errno is set.
7152  */
7153 int
7154 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7155 {
7156         struct rte_flow_error error;
7157         struct mlx5_priv *priv = dev->data->dev_private;
7158         struct mlx5_shared_action_rss *action;
7159         int ret = 0;
7160         uint32_t idx;
7161
7162         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7163                       priv->rss_shared_actions, idx, action, next) {
7164                 ret |= mlx5_shared_action_destroy(dev,
7165                        (struct rte_flow_shared_action *)(uintptr_t)idx, &error);
7166         }
7167         return ret;
7168 }
7169
7170 #ifndef HAVE_MLX5DV_DR
7171 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
7172 #else
7173 #define MLX5_DOMAIN_SYNC_FLOW \
7174         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
7175 #endif
7176
7177 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
7178 {
7179         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
7180         const struct mlx5_flow_driver_ops *fops;
7181         int ret;
7182         struct rte_flow_attr attr = { .transfer = 0 };
7183
7184         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7185         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
7186         if (ret > 0)
7187                 ret = -ret;
7188         return ret;
7189 }
7190
7191 /**
7192  * tunnel offload functionalilty is defined for DV environment only
7193  */
7194 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7195 __extension__
7196 union tunnel_offload_mark {
7197         uint32_t val;
7198         struct {
7199                 uint32_t app_reserve:8;
7200                 uint32_t table_id:15;
7201                 uint32_t transfer:1;
7202                 uint32_t _unused_:8;
7203         };
7204 };
7205
7206 static bool
7207 mlx5_access_tunnel_offload_db
7208         (struct rte_eth_dev *dev,
7209          bool (*match)(struct rte_eth_dev *,
7210                        struct mlx5_flow_tunnel *, const void *),
7211          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7212          void (*miss)(struct rte_eth_dev *, void *),
7213          void *ctx, bool lock_op);
7214
7215 static int
7216 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
7217                              struct rte_flow *flow,
7218                              const struct rte_flow_attr *attr,
7219                              const struct rte_flow_action *app_actions,
7220                              uint32_t flow_idx,
7221                              struct tunnel_default_miss_ctx *ctx,
7222                              struct rte_flow_error *error)
7223 {
7224         struct mlx5_priv *priv = dev->data->dev_private;
7225         struct mlx5_flow *dev_flow;
7226         struct rte_flow_attr miss_attr = *attr;
7227         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
7228         const struct rte_flow_item miss_items[2] = {
7229                 {
7230                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7231                         .spec = NULL,
7232                         .last = NULL,
7233                         .mask = NULL
7234                 },
7235                 {
7236                         .type = RTE_FLOW_ITEM_TYPE_END,
7237                         .spec = NULL,
7238                         .last = NULL,
7239                         .mask = NULL
7240                 }
7241         };
7242         union tunnel_offload_mark mark_id;
7243         struct rte_flow_action_mark miss_mark;
7244         struct rte_flow_action miss_actions[3] = {
7245                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
7246                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
7247         };
7248         const struct rte_flow_action_jump *jump_data;
7249         uint32_t i, flow_table = 0; /* prevent compilation warning */
7250         struct flow_grp_info grp_info = {
7251                 .external = 1,
7252                 .transfer = attr->transfer,
7253                 .fdb_def_rule = !!priv->fdb_def_rule,
7254                 .std_tbl_fix = 0,
7255         };
7256         int ret;
7257
7258         if (!attr->transfer) {
7259                 uint32_t q_size;
7260
7261                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
7262                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
7263                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
7264                                          0, SOCKET_ID_ANY);
7265                 if (!ctx->queue)
7266                         return rte_flow_error_set
7267                                 (error, ENOMEM,
7268                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7269                                 NULL, "invalid default miss RSS");
7270                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7271                 ctx->action_rss.level = 0,
7272                 ctx->action_rss.types = priv->rss_conf.rss_hf,
7273                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
7274                 ctx->action_rss.queue_num = priv->reta_idx_n,
7275                 ctx->action_rss.key = priv->rss_conf.rss_key,
7276                 ctx->action_rss.queue = ctx->queue;
7277                 if (!priv->reta_idx_n || !priv->rxqs_n)
7278                         return rte_flow_error_set
7279                                 (error, EINVAL,
7280                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7281                                 NULL, "invalid port configuration");
7282                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7283                         ctx->action_rss.types = 0;
7284                 for (i = 0; i != priv->reta_idx_n; ++i)
7285                         ctx->queue[i] = (*priv->reta_idx)[i];
7286         } else {
7287                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
7288                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
7289         }
7290         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
7291         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
7292         jump_data = app_actions->conf;
7293         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
7294         miss_attr.group = jump_data->group;
7295         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
7296                                        &flow_table, &grp_info, error);
7297         if (ret)
7298                 return rte_flow_error_set(error, EINVAL,
7299                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7300                                           NULL, "invalid tunnel id");
7301         mark_id.app_reserve = 0;
7302         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
7303         mark_id.transfer = !!attr->transfer;
7304         mark_id._unused_ = 0;
7305         miss_mark.id = mark_id.val;
7306         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
7307                                     miss_items, miss_actions, flow_idx, error);
7308         if (!dev_flow)
7309                 return -rte_errno;
7310         dev_flow->flow = flow;
7311         dev_flow->external = true;
7312         dev_flow->tunnel = tunnel;
7313         /* Subflow object was created, we must include one in the list. */
7314         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7315                       dev_flow->handle, next);
7316         DRV_LOG(DEBUG,
7317                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
7318                 dev->data->port_id, tunnel->app_tunnel.type,
7319                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
7320         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
7321                                   miss_actions, error);
7322         if (!ret)
7323                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
7324                                                   error);
7325
7326         return ret;
7327 }
7328
7329 static const struct mlx5_flow_tbl_data_entry  *
7330 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7331 {
7332         struct mlx5_priv *priv = dev->data->dev_private;
7333         struct mlx5_dev_ctx_shared *sh = priv->sh;
7334         struct mlx5_hlist_entry *he;
7335         union tunnel_offload_mark mbits = { .val = mark };
7336         union mlx5_flow_tbl_key table_key = {
7337                 {
7338                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7339                         .dummy = 0,
7340                         .domain = !!mbits.transfer,
7341                         .direction = 0,
7342                 }
7343         };
7344         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
7345         return he ?
7346                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7347 }
7348
7349 static void
7350 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
7351                                    struct mlx5_hlist_entry *entry)
7352 {
7353         struct mlx5_dev_ctx_shared *sh = list->ctx;
7354         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7355
7356         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7357                         tunnel_flow_tbl_to_id(tte->flow_table));
7358         mlx5_free(tte);
7359 }
7360
7361 static struct mlx5_hlist_entry *
7362 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
7363                                    uint64_t key __rte_unused,
7364                                    void *ctx __rte_unused)
7365 {
7366         struct mlx5_dev_ctx_shared *sh = list->ctx;
7367         struct tunnel_tbl_entry *tte;
7368
7369         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7370                           sizeof(*tte), 0,
7371                           SOCKET_ID_ANY);
7372         if (!tte)
7373                 goto err;
7374         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7375                           &tte->flow_table);
7376         if (tte->flow_table >= MLX5_MAX_TABLES) {
7377                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
7378                         tte->flow_table);
7379                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7380                                 tte->flow_table);
7381                 goto err;
7382         } else if (!tte->flow_table) {
7383                 goto err;
7384         }
7385         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7386         return &tte->hash;
7387 err:
7388         if (tte)
7389                 mlx5_free(tte);
7390         return NULL;
7391 }
7392
7393 static uint32_t
7394 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7395                                 const struct mlx5_flow_tunnel *tunnel,
7396                                 uint32_t group, uint32_t *table,
7397                                 struct rte_flow_error *error)
7398 {
7399         struct mlx5_hlist_entry *he;
7400         struct tunnel_tbl_entry *tte;
7401         union tunnel_tbl_key key = {
7402                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7403                 .group = group
7404         };
7405         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7406         struct mlx5_hlist *group_hash;
7407
7408         group_hash = tunnel ? tunnel->groups : thub->groups;
7409         he = mlx5_hlist_register(group_hash, key.val, NULL);
7410         if (!he)
7411                 return rte_flow_error_set(error, EINVAL,
7412                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7413                                           NULL,
7414                                           "tunnel group index not supported");
7415         tte = container_of(he, typeof(*tte), hash);
7416         *table = tte->flow_table;
7417         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7418                 dev->data->port_id, key.tunnel_id, group, *table);
7419         return 0;
7420 }
7421
7422 static void
7423 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
7424                       struct mlx5_flow_tunnel *tunnel)
7425 {
7426         struct mlx5_priv *priv = dev->data->dev_private;
7427         struct mlx5_indexed_pool *ipool;
7428
7429         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
7430                 dev->data->port_id, tunnel->tunnel_id);
7431         mlx5_hlist_destroy(tunnel->groups);
7432         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7433         mlx5_ipool_free(ipool, tunnel->tunnel_id);
7434 }
7435
7436 static bool
7437 mlx5_access_tunnel_offload_db
7438         (struct rte_eth_dev *dev,
7439          bool (*match)(struct rte_eth_dev *,
7440                        struct mlx5_flow_tunnel *, const void *),
7441          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7442          void (*miss)(struct rte_eth_dev *, void *),
7443          void *ctx, bool lock_op)
7444 {
7445         bool verdict = false;
7446         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7447         struct mlx5_flow_tunnel *tunnel;
7448
7449         rte_spinlock_lock(&thub->sl);
7450         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
7451                 verdict = match(dev, tunnel, (const void *)ctx);
7452                 if (verdict)
7453                         break;
7454         }
7455         if (!lock_op)
7456                 rte_spinlock_unlock(&thub->sl);
7457         if (verdict && hit)
7458                 hit(dev, tunnel, ctx);
7459         if (!verdict && miss)
7460                 miss(dev, ctx);
7461         if (lock_op)
7462                 rte_spinlock_unlock(&thub->sl);
7463
7464         return verdict;
7465 }
7466
7467 struct tunnel_db_find_tunnel_id_ctx {
7468         uint32_t tunnel_id;
7469         struct mlx5_flow_tunnel *tunnel;
7470 };
7471
7472 static bool
7473 find_tunnel_id_match(struct rte_eth_dev *dev,
7474                      struct mlx5_flow_tunnel *tunnel, const void *x)
7475 {
7476         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7477
7478         RTE_SET_USED(dev);
7479         return tunnel->tunnel_id == ctx->tunnel_id;
7480 }
7481
7482 static void
7483 find_tunnel_id_hit(struct rte_eth_dev *dev,
7484                    struct mlx5_flow_tunnel *tunnel, void *x)
7485 {
7486         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7487         RTE_SET_USED(dev);
7488         ctx->tunnel = tunnel;
7489 }
7490
7491 static struct mlx5_flow_tunnel *
7492 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
7493 {
7494         struct tunnel_db_find_tunnel_id_ctx ctx = {
7495                 .tunnel_id = id,
7496         };
7497
7498         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
7499                                       find_tunnel_id_hit, NULL, &ctx, true);
7500
7501         return ctx.tunnel;
7502 }
7503
7504 static struct mlx5_flow_tunnel *
7505 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
7506                           const struct rte_flow_tunnel *app_tunnel)
7507 {
7508         struct mlx5_priv *priv = dev->data->dev_private;
7509         struct mlx5_indexed_pool *ipool;
7510         struct mlx5_flow_tunnel *tunnel;
7511         uint32_t id;
7512
7513         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7514         tunnel = mlx5_ipool_zmalloc(ipool, &id);
7515         if (!tunnel)
7516                 return NULL;
7517         if (id >= MLX5_MAX_TUNNELS) {
7518                 mlx5_ipool_free(ipool, id);
7519                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
7520                 return NULL;
7521         }
7522         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
7523                                            mlx5_flow_tunnel_grp2tbl_create_cb,
7524                                            NULL,
7525                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
7526         if (!tunnel->groups) {
7527                 mlx5_ipool_free(ipool, id);
7528                 return NULL;
7529         }
7530         tunnel->groups->ctx = priv->sh;
7531         /* initiate new PMD tunnel */
7532         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
7533         tunnel->tunnel_id = id;
7534         tunnel->action.type = (typeof(tunnel->action.type))
7535                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
7536         tunnel->action.conf = tunnel;
7537         tunnel->item.type = (typeof(tunnel->item.type))
7538                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
7539         tunnel->item.spec = tunnel;
7540         tunnel->item.last = NULL;
7541         tunnel->item.mask = NULL;
7542
7543         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
7544                 dev->data->port_id, tunnel->tunnel_id);
7545
7546         return tunnel;
7547 }
7548
7549 struct tunnel_db_get_tunnel_ctx {
7550         const struct rte_flow_tunnel *app_tunnel;
7551         struct mlx5_flow_tunnel *tunnel;
7552 };
7553
7554 static bool get_tunnel_match(struct rte_eth_dev *dev,
7555                              struct mlx5_flow_tunnel *tunnel, const void *x)
7556 {
7557         const struct tunnel_db_get_tunnel_ctx *ctx = x;
7558
7559         RTE_SET_USED(dev);
7560         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
7561                        sizeof(*ctx->app_tunnel));
7562 }
7563
7564 static void get_tunnel_hit(struct rte_eth_dev *dev,
7565                            struct mlx5_flow_tunnel *tunnel, void *x)
7566 {
7567         /* called under tunnel spinlock protection */
7568         struct tunnel_db_get_tunnel_ctx *ctx = x;
7569
7570         RTE_SET_USED(dev);
7571         tunnel->refctn++;
7572         ctx->tunnel = tunnel;
7573 }
7574
7575 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
7576 {
7577         /* called under tunnel spinlock protection */
7578         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7579         struct tunnel_db_get_tunnel_ctx *ctx = x;
7580
7581         rte_spinlock_unlock(&thub->sl);
7582         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
7583         ctx->tunnel->refctn = 1;
7584         rte_spinlock_lock(&thub->sl);
7585         if (ctx->tunnel)
7586                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
7587 }
7588
7589
7590 static int
7591 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
7592                      const struct rte_flow_tunnel *app_tunnel,
7593                      struct mlx5_flow_tunnel **tunnel)
7594 {
7595         struct tunnel_db_get_tunnel_ctx ctx = {
7596                 .app_tunnel = app_tunnel,
7597         };
7598
7599         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
7600                                       get_tunnel_miss, &ctx, true);
7601         *tunnel = ctx.tunnel;
7602         return ctx.tunnel ? 0 : -ENOMEM;
7603 }
7604
7605 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
7606 {
7607         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
7608
7609         if (!thub)
7610                 return;
7611         if (!LIST_EMPTY(&thub->tunnels))
7612                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
7613         mlx5_hlist_destroy(thub->groups);
7614         mlx5_free(thub);
7615 }
7616
7617 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
7618 {
7619         int err;
7620         struct mlx5_flow_tunnel_hub *thub;
7621
7622         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
7623                            0, SOCKET_ID_ANY);
7624         if (!thub)
7625                 return -ENOMEM;
7626         LIST_INIT(&thub->tunnels);
7627         rte_spinlock_init(&thub->sl);
7628         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
7629                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
7630                                          NULL,
7631                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
7632         if (!thub->groups) {
7633                 err = -rte_errno;
7634                 goto err;
7635         }
7636         thub->groups->ctx = sh;
7637         sh->tunnel_hub = thub;
7638
7639         return 0;
7640
7641 err:
7642         if (thub->groups)
7643                 mlx5_hlist_destroy(thub->groups);
7644         if (thub)
7645                 mlx5_free(thub);
7646         return err;
7647 }
7648
7649 static inline bool
7650 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
7651                           struct rte_flow_tunnel *tunnel,
7652                           const char *err_msg)
7653 {
7654         err_msg = NULL;
7655         if (!is_tunnel_offload_active(dev)) {
7656                 err_msg = "tunnel offload was not activated";
7657                 goto out;
7658         } else if (!tunnel) {
7659                 err_msg = "no application tunnel";
7660                 goto out;
7661         }
7662
7663         switch (tunnel->type) {
7664         default:
7665                 err_msg = "unsupported tunnel type";
7666                 goto out;
7667         case RTE_FLOW_ITEM_TYPE_VXLAN:
7668                 break;
7669         }
7670
7671 out:
7672         return !err_msg;
7673 }
7674
7675 static int
7676 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
7677                     struct rte_flow_tunnel *app_tunnel,
7678                     struct rte_flow_action **actions,
7679                     uint32_t *num_of_actions,
7680                     struct rte_flow_error *error)
7681 {
7682         int ret;
7683         struct mlx5_flow_tunnel *tunnel;
7684         const char *err_msg = NULL;
7685         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7686
7687         if (!verdict)
7688                 return rte_flow_error_set(error, EINVAL,
7689                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7690                                           err_msg);
7691         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7692         if (ret < 0) {
7693                 return rte_flow_error_set(error, ret,
7694                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7695                                           "failed to initialize pmd tunnel");
7696         }
7697         *actions = &tunnel->action;
7698         *num_of_actions = 1;
7699         return 0;
7700 }
7701
7702 static int
7703 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
7704                        struct rte_flow_tunnel *app_tunnel,
7705                        struct rte_flow_item **items,
7706                        uint32_t *num_of_items,
7707                        struct rte_flow_error *error)
7708 {
7709         int ret;
7710         struct mlx5_flow_tunnel *tunnel;
7711         const char *err_msg = NULL;
7712         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7713
7714         if (!verdict)
7715                 return rte_flow_error_set(error, EINVAL,
7716                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7717                                           err_msg);
7718         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7719         if (ret < 0) {
7720                 return rte_flow_error_set(error, ret,
7721                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7722                                           "failed to initialize pmd tunnel");
7723         }
7724         *items = &tunnel->item;
7725         *num_of_items = 1;
7726         return 0;
7727 }
7728
7729 struct tunnel_db_element_release_ctx {
7730         struct rte_flow_item *items;
7731         struct rte_flow_action *actions;
7732         uint32_t num_elements;
7733         struct rte_flow_error *error;
7734         int ret;
7735 };
7736
7737 static bool
7738 tunnel_element_release_match(struct rte_eth_dev *dev,
7739                              struct mlx5_flow_tunnel *tunnel, const void *x)
7740 {
7741         const struct tunnel_db_element_release_ctx *ctx = x;
7742
7743         RTE_SET_USED(dev);
7744         if (ctx->num_elements != 1)
7745                 return false;
7746         else if (ctx->items)
7747                 return ctx->items == &tunnel->item;
7748         else if (ctx->actions)
7749                 return ctx->actions == &tunnel->action;
7750
7751         return false;
7752 }
7753
7754 static void
7755 tunnel_element_release_hit(struct rte_eth_dev *dev,
7756                            struct mlx5_flow_tunnel *tunnel, void *x)
7757 {
7758         struct tunnel_db_element_release_ctx *ctx = x;
7759         ctx->ret = 0;
7760         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7761                 mlx5_flow_tunnel_free(dev, tunnel);
7762 }
7763
7764 static void
7765 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
7766 {
7767         struct tunnel_db_element_release_ctx *ctx = x;
7768         RTE_SET_USED(dev);
7769         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
7770                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7771                                       "invalid argument");
7772 }
7773
7774 static int
7775 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
7776                        struct rte_flow_item *pmd_items,
7777                        uint32_t num_items, struct rte_flow_error *err)
7778 {
7779         struct tunnel_db_element_release_ctx ctx = {
7780                 .items = pmd_items,
7781                 .actions = NULL,
7782                 .num_elements = num_items,
7783                 .error = err,
7784         };
7785
7786         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7787                                       tunnel_element_release_hit,
7788                                       tunnel_element_release_miss, &ctx, false);
7789
7790         return ctx.ret;
7791 }
7792
7793 static int
7794 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
7795                          struct rte_flow_action *pmd_actions,
7796                          uint32_t num_actions, struct rte_flow_error *err)
7797 {
7798         struct tunnel_db_element_release_ctx ctx = {
7799                 .items = NULL,
7800                 .actions = pmd_actions,
7801                 .num_elements = num_actions,
7802                 .error = err,
7803         };
7804
7805         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7806                                       tunnel_element_release_hit,
7807                                       tunnel_element_release_miss, &ctx, false);
7808
7809         return ctx.ret;
7810 }
7811
7812 static int
7813 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
7814                                   struct rte_mbuf *m,
7815                                   struct rte_flow_restore_info *info,
7816                                   struct rte_flow_error *err)
7817 {
7818         uint64_t ol_flags = m->ol_flags;
7819         const struct mlx5_flow_tbl_data_entry *tble;
7820         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
7821
7822         if (!is_tunnel_offload_active(dev)) {
7823                 info->flags = 0;
7824                 return 0;
7825         }
7826
7827         if ((ol_flags & mask) != mask)
7828                 goto err;
7829         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
7830         if (!tble) {
7831                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
7832                         dev->data->port_id, m->hash.fdir.hi);
7833                 goto err;
7834         }
7835         MLX5_ASSERT(tble->tunnel);
7836         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
7837         info->group_id = tble->group_id;
7838         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
7839                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
7840                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
7841
7842         return 0;
7843
7844 err:
7845         return rte_flow_error_set(err, EINVAL,
7846                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7847                                   "failed to get restore info");
7848 }
7849
7850 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
7851 static int
7852 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
7853                            __rte_unused struct rte_flow_tunnel *app_tunnel,
7854                            __rte_unused struct rte_flow_action **actions,
7855                            __rte_unused uint32_t *num_of_actions,
7856                            __rte_unused struct rte_flow_error *error)
7857 {
7858         return -ENOTSUP;
7859 }
7860
7861 static int
7862 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
7863                        __rte_unused struct rte_flow_tunnel *app_tunnel,
7864                        __rte_unused struct rte_flow_item **items,
7865                        __rte_unused uint32_t *num_of_items,
7866                        __rte_unused struct rte_flow_error *error)
7867 {
7868         return -ENOTSUP;
7869 }
7870
7871 static int
7872 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
7873                               __rte_unused struct rte_flow_item *pmd_items,
7874                               __rte_unused uint32_t num_items,
7875                               __rte_unused struct rte_flow_error *err)
7876 {
7877         return -ENOTSUP;
7878 }
7879
7880 static int
7881 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
7882                                 __rte_unused struct rte_flow_action *pmd_action,
7883                                 __rte_unused uint32_t num_actions,
7884                                 __rte_unused struct rte_flow_error *err)
7885 {
7886         return -ENOTSUP;
7887 }
7888
7889 static int
7890 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
7891                                   __rte_unused struct rte_mbuf *m,
7892                                   __rte_unused struct rte_flow_restore_info *i,
7893                                   __rte_unused struct rte_flow_error *err)
7894 {
7895         return -ENOTSUP;
7896 }
7897
7898 static int
7899 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
7900                              __rte_unused struct rte_flow *flow,
7901                              __rte_unused const struct rte_flow_attr *attr,
7902                              __rte_unused const struct rte_flow_action *actions,
7903                              __rte_unused uint32_t flow_idx,
7904                              __rte_unused struct tunnel_default_miss_ctx *ctx,
7905                              __rte_unused struct rte_flow_error *error)
7906 {
7907         return -ENOTSUP;
7908 }
7909
7910 static struct mlx5_flow_tunnel *
7911 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
7912                     __rte_unused uint32_t id)
7913 {
7914         return NULL;
7915 }
7916
7917 static void
7918 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
7919                       __rte_unused struct mlx5_flow_tunnel *tunnel)
7920 {
7921 }
7922
7923 static uint32_t
7924 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
7925                                 __rte_unused const struct mlx5_flow_tunnel *t,
7926                                 __rte_unused uint32_t group,
7927                                 __rte_unused uint32_t *table,
7928                                 struct rte_flow_error *error)
7929 {
7930         return rte_flow_error_set(error, ENOTSUP,
7931                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7932                                   "tunnel offload requires DV support");
7933 }
7934
7935 void
7936 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
7937                         __rte_unused  uint16_t port_id)
7938 {
7939 }
7940 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
7941