drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              struct tunnel_default_miss_ctx *ctx,
  54                              struct rte_flow_error *error);
  55 static struct mlx5_flow_tunnel *
  56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  57 static void
  58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  59 static uint32_t
  60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  61                                 const struct mlx5_flow_tunnel *tunnel,
  62                                 uint32_t group, uint32_t *table,
  63                                 struct rte_flow_error *error);
  64
  65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  66 static void mlx5_flow_pop_thread_workspace(void);
  67
  68
  69 /** Device flow drivers. */
  70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  71
  72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  73
  74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  75         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  76 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  77         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  78 #endif
  79         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  80         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  81 };
  82
  83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  85         (const int []){ \
  86                 __VA_ARGS__, 0, \
  87         }
  88
  89 /** Node object of input graph for mlx5_flow_expand_rss(). */
  90 struct mlx5_flow_expand_node {
  91         const int *const next;
  92         /**<
  93          * List of next node indexes. Index 0 is interpreted as a terminator.
  94          */
  95         const enum rte_flow_item_type type;
  96         /**< Pattern item type of current node. */
  97         uint64_t rss_types;
  98         /**<
  99          * RSS types bit-field associated with this node
 100          * (see ETH_RSS_* definitions).
 101          */
 102 };
 103
 104 /** Object returned by mlx5_flow_expand_rss(). */
 105 struct mlx5_flow_expand_rss {
 106         uint32_t entries;
 107         /**< Number of entries @p patterns and @p priorities. */
 108         struct {
 109                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 110                 uint32_t priority; /**< Priority offset for each expansion. */
 111         } entry[];
 112 };
 113
 114 static enum rte_flow_item_type
 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 116 {
 117         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 118         uint16_t ether_type = 0;
 119         uint16_t ether_type_m;
 120         uint8_t ip_next_proto = 0;
 121         uint8_t ip_next_proto_m;
 122
 123         if (item == NULL || item->spec == NULL)
 124                 return ret;
 125         switch (item->type) {
 126         case RTE_FLOW_ITEM_TYPE_ETH:
 127                 if (item->mask)
 128                         ether_type_m = ((const struct rte_flow_item_eth *)
 129                                                 (item->mask))->type;
 130                 else
 131                         ether_type_m = rte_flow_item_eth_mask.type;
 132                 if (ether_type_m != RTE_BE16(0xFFFF))
 133                         break;
 134                 ether_type = ((const struct rte_flow_item_eth *)
 135                                 (item->spec))->type;
 136                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 139                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 140                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 141                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 142                 else
 143                         ret = RTE_FLOW_ITEM_TYPE_END;
 144                 break;
 145         case RTE_FLOW_ITEM_TYPE_VLAN:
 146                 if (item->mask)
 147                         ether_type_m = ((const struct rte_flow_item_vlan *)
 148                                                 (item->mask))->inner_type;
 149                 else
 150                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 151                 if (ether_type_m != RTE_BE16(0xFFFF))
 152                         break;
 153                 ether_type = ((const struct rte_flow_item_vlan *)
 154                                 (item->spec))->inner_type;
 155                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 156                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 157                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 159                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 160                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV4:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 167                                         (item->mask))->hdr.next_proto_id;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 174                                 (item->spec))->hdr.next_proto_id;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         case RTE_FLOW_ITEM_TYPE_IPV6:
 187                 if (item->mask)
 188                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 189                                                 (item->mask))->hdr.proto;
 190                 else
 191                         ip_next_proto_m =
 192                                 rte_flow_item_ipv6_mask.hdr.proto;
 193                 if (ip_next_proto_m != 0xFF)
 194                         break;
 195                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 196                                 (item->spec))->hdr.proto;
 197                 if (ip_next_proto == IPPROTO_UDP)
 198                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 199                 else if (ip_next_proto == IPPROTO_TCP)
 200                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 201                 else if (ip_next_proto == IPPROTO_IP)
 202                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 203                 else if (ip_next_proto == IPPROTO_IPV6)
 204                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 205                 else
 206                         ret = RTE_FLOW_ITEM_TYPE_END;
 207                 break;
 208         default:
 209                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 210                 break;
 211         }
 212         return ret;
 213 }
 214
 215 #define MLX5_RSS_EXP_ELT_N 8
 216
 217 /**
 218  * Expand RSS flows into several possible flows according to the RSS hash
 219  * fields requested and the driver capabilities.
 220  *
 221  * @param[out] buf
 222  *   Buffer to store the result expansion.
 223  * @param[in] size
 224  *   Buffer size in bytes. If 0, @p buf can be NULL.
 225  * @param[in] pattern
 226  *   User flow pattern.
 227  * @param[in] types
 228  *   RSS types to expand (see ETH_RSS_* definitions).
 229  * @param[in] graph
 230  *   Input graph to expand @p pattern according to @p types.
 231  * @param[in] graph_root_index
 232  *   Index of root node in @p graph, typically 0.
 233  *
 234  * @return
 235  *   A positive value representing the size of @p buf in bytes regardless of
 236  *   @p size on success, a negative errno value otherwise and rte_errno is
 237  *   set, the following errors are defined:
 238  *
 239  *   -E2BIG: graph-depth @p graph is too deep.
 240  */
 241 static int
 242 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 243                      const struct rte_flow_item *pattern, uint64_t types,
 244                      const struct mlx5_flow_expand_node graph[],
 245                      int graph_root_index)
 246 {
 247         const struct rte_flow_item *item;
 248         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 249         const int *next_node;
 250         const int *stack[MLX5_RSS_EXP_ELT_N];
 251         int stack_pos = 0;
 252         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 253         unsigned int i;
 254         size_t lsize;
 255         size_t user_pattern_size = 0;
 256         void *addr = NULL;
 257         const struct mlx5_flow_expand_node *next = NULL;
 258         struct rte_flow_item missed_item;
 259         int missed = 0;
 260         int elt = 0;
 261         const struct rte_flow_item *last_item = NULL;
 262
 263         memset(&missed_item, 0, sizeof(missed_item));
 264         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 265                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 266         if (lsize <= size) {
 267                 buf->entry[0].priority = 0;
 268                 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 269                 buf->entries = 0;
 270                 addr = buf->entry[0].pattern;
 271         }
 272         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 273                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 274                         last_item = item;
 275                 for (i = 0; node->next && node->next[i]; ++i) {
 276                         next = &graph[node->next[i]];
 277                         if (next->type == item->type)
 278                                 break;
 279                 }
 280                 if (next)
 281                         node = next;
 282                 user_pattern_size += sizeof(*item);
 283         }
 284         user_pattern_size += sizeof(*item); /* Handle END item. */
 285         lsize += user_pattern_size;
 286         /* Copy the user pattern in the first entry of the buffer. */
 287         if (lsize <= size) {
 288                 rte_memcpy(addr, pattern, user_pattern_size);
 289                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 290                 buf->entries = 1;
 291         }
 292         /* Start expanding. */
 293         memset(flow_items, 0, sizeof(flow_items));
 294         user_pattern_size -= sizeof(*item);
 295         /*
 296          * Check if the last valid item has spec set, need complete pattern,
 297          * and the pattern can be used for expansion.
 298          */
 299         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 300         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 301                 /* Item type END indicates expansion is not required. */
 302                 return lsize;
 303         }
 304         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 305                 next = NULL;
 306                 missed = 1;
 307                 for (i = 0; node->next && node->next[i]; ++i) {
 308                         next = &graph[node->next[i]];
 309                         if (next->type == missed_item.type) {
 310                                 flow_items[0].type = missed_item.type;
 311                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 312                                 break;
 313                         }
 314                         next = NULL;
 315                 }
 316         }
 317         if (next && missed) {
 318                 elt = 2; /* missed item + item end. */
 319                 node = next;
 320                 lsize += elt * sizeof(*item) + user_pattern_size;
 321                 if ((node->rss_types & types) && lsize <= size) {
 322                         buf->entry[buf->entries].priority = 1;
 323                         buf->entry[buf->entries].pattern = addr;
 324                         buf->entries++;
 325                         rte_memcpy(addr, buf->entry[0].pattern,
 326                                    user_pattern_size);
 327                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 328                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 329                         addr = (void *)(((uintptr_t)addr) +
 330                                         elt * sizeof(*item));
 331                 }
 332         }
 333         memset(flow_items, 0, sizeof(flow_items));
 334         next_node = node->next;
 335         stack[stack_pos] = next_node;
 336         node = next_node ? &graph[*next_node] : NULL;
 337         while (node) {
 338                 flow_items[stack_pos].type = node->type;
 339                 if (node->rss_types & types) {
 340                         /*
 341                          * compute the number of items to copy from the
 342                          * expansion and copy it.
 343                          * When the stack_pos is 0, there are 1 element in it,
 344                          * plus the addition END item.
 345                          */
 346                         elt = stack_pos + 2;
 347                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 348                         lsize += elt * sizeof(*item) + user_pattern_size;
 349                         if (lsize <= size) {
 350                                 size_t n = elt * sizeof(*item);
 351
 352                                 buf->entry[buf->entries].priority =
 353                                         stack_pos + 1 + missed;
 354                                 buf->entry[buf->entries].pattern = addr;
 355                                 buf->entries++;
 356                                 rte_memcpy(addr, buf->entry[0].pattern,
 357                                            user_pattern_size);
 358                                 addr = (void *)(((uintptr_t)addr) +
 359                                                 user_pattern_size);
 360                                 rte_memcpy(addr, &missed_item,
 361                                            missed * sizeof(*item));
 362                                 addr = (void *)(((uintptr_t)addr) +
 363                                         missed * sizeof(*item));
 364                                 rte_memcpy(addr, flow_items, n);
 365                                 addr = (void *)(((uintptr_t)addr) + n);
 366                         }
 367                 }
 368                 /* Go deeper. */
 369                 if (node->next) {
 370                         next_node = node->next;
 371                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 372                                 rte_errno = E2BIG;
 373                                 return -rte_errno;
 374                         }
 375                         stack[stack_pos] = next_node;
 376                 } else if (*(next_node + 1)) {
 377                         /* Follow up with the next possibility. */
 378                         ++next_node;
 379                 } else {
 380                         /* Move to the next path. */
 381                         if (stack_pos)
 382                                 next_node = stack[--stack_pos];
 383                         next_node++;
 384                         stack[stack_pos] = next_node;
 385                 }
 386                 node = *next_node ? &graph[*next_node] : NULL;
 387         };
 388         /* no expanded flows but we have missed item, create one rule for it */
 389         if (buf->entries == 1 && missed != 0) {
 390                 elt = 2;
 391                 lsize += elt * sizeof(*item) + user_pattern_size;
 392                 if (lsize <= size) {
 393                         buf->entry[buf->entries].priority = 1;
 394                         buf->entry[buf->entries].pattern = addr;
 395                         buf->entries++;
 396                         flow_items[0].type = missed_item.type;
 397                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 398                         rte_memcpy(addr, buf->entry[0].pattern,
 399                                    user_pattern_size);
 400                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 401                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 402                 }
 403         }
 404         return lsize;
 405 }
 406
 407 enum mlx5_expansion {
 408         MLX5_EXPANSION_ROOT,
 409         MLX5_EXPANSION_ROOT_OUTER,
 410         MLX5_EXPANSION_ROOT_ETH_VLAN,
 411         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 412         MLX5_EXPANSION_OUTER_ETH,
 413         MLX5_EXPANSION_OUTER_ETH_VLAN,
 414         MLX5_EXPANSION_OUTER_VLAN,
 415         MLX5_EXPANSION_OUTER_IPV4,
 416         MLX5_EXPANSION_OUTER_IPV4_UDP,
 417         MLX5_EXPANSION_OUTER_IPV4_TCP,
 418         MLX5_EXPANSION_OUTER_IPV6,
 419         MLX5_EXPANSION_OUTER_IPV6_UDP,
 420         MLX5_EXPANSION_OUTER_IPV6_TCP,
 421         MLX5_EXPANSION_VXLAN,
 422         MLX5_EXPANSION_VXLAN_GPE,
 423         MLX5_EXPANSION_GRE,
 424         MLX5_EXPANSION_MPLS,
 425         MLX5_EXPANSION_ETH,
 426         MLX5_EXPANSION_ETH_VLAN,
 427         MLX5_EXPANSION_VLAN,
 428         MLX5_EXPANSION_IPV4,
 429         MLX5_EXPANSION_IPV4_UDP,
 430         MLX5_EXPANSION_IPV4_TCP,
 431         MLX5_EXPANSION_IPV6,
 432         MLX5_EXPANSION_IPV6_UDP,
 433         MLX5_EXPANSION_IPV6_TCP,
 434 };
 435
 436 /** Supported expansion of items. */
 437 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 438         [MLX5_EXPANSION_ROOT] = {
 439                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 440                                                   MLX5_EXPANSION_IPV4,
 441                                                   MLX5_EXPANSION_IPV6),
 442                 .type = RTE_FLOW_ITEM_TYPE_END,
 443         },
 444         [MLX5_EXPANSION_ROOT_OUTER] = {
 445                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 446                                                   MLX5_EXPANSION_OUTER_IPV4,
 447                                                   MLX5_EXPANSION_OUTER_IPV6),
 448                 .type = RTE_FLOW_ITEM_TYPE_END,
 449         },
 450         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 451                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 452                 .type = RTE_FLOW_ITEM_TYPE_END,
 453         },
 454         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 455                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 456                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 457                 .type = RTE_FLOW_ITEM_TYPE_END,
 458         },
 459         [MLX5_EXPANSION_OUTER_ETH] = {
 460                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 461                                                   MLX5_EXPANSION_OUTER_IPV6,
 462                                                   MLX5_EXPANSION_MPLS),
 463                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 464                 .rss_types = 0,
 465         },
 466         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 467                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 468                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 469                 .rss_types = 0,
 470         },
 471         [MLX5_EXPANSION_OUTER_VLAN] = {
 472                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 473                                                   MLX5_EXPANSION_OUTER_IPV6),
 474                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 475         },
 476         [MLX5_EXPANSION_OUTER_IPV4] = {
 477                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 478                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 479                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 480                          MLX5_EXPANSION_GRE,
 481                          MLX5_EXPANSION_IPV4,
 482                          MLX5_EXPANSION_IPV6),
 483                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 484                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 485                         ETH_RSS_NONFRAG_IPV4_OTHER,
 486         },
 487         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 488                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 489                                                   MLX5_EXPANSION_VXLAN_GPE),
 490                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 491                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 492         },
 493         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 494                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 495                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 496         },
 497         [MLX5_EXPANSION_OUTER_IPV6] = {
 498                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 499                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 500                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 501                          MLX5_EXPANSION_IPV4,
 502                          MLX5_EXPANSION_IPV6),
 503                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 504                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 505                         ETH_RSS_NONFRAG_IPV6_OTHER,
 506         },
 507         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 508                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 509                                                   MLX5_EXPANSION_VXLAN_GPE),
 510                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 511                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 512         },
 513         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 514                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 515                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 516         },
 517         [MLX5_EXPANSION_VXLAN] = {
 518                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 519                                                   MLX5_EXPANSION_IPV4,
 520                                                   MLX5_EXPANSION_IPV6),
 521                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 522         },
 523         [MLX5_EXPANSION_VXLAN_GPE] = {
 524                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 525                                                   MLX5_EXPANSION_IPV4,
 526                                                   MLX5_EXPANSION_IPV6),
 527                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 528         },
 529         [MLX5_EXPANSION_GRE] = {
 530                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 531                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 532         },
 533         [MLX5_EXPANSION_MPLS] = {
 534                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 535                                                   MLX5_EXPANSION_IPV6),
 536                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 537         },
 538         [MLX5_EXPANSION_ETH] = {
 539                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 540                                                   MLX5_EXPANSION_IPV6),
 541                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 542         },
 543         [MLX5_EXPANSION_ETH_VLAN] = {
 544                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 545                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 546         },
 547         [MLX5_EXPANSION_VLAN] = {
 548                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 549                                                   MLX5_EXPANSION_IPV6),
 550                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 551         },
 552         [MLX5_EXPANSION_IPV4] = {
 553                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 554                                                   MLX5_EXPANSION_IPV4_TCP),
 555                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 556                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 557                         ETH_RSS_NONFRAG_IPV4_OTHER,
 558         },
 559         [MLX5_EXPANSION_IPV4_UDP] = {
 560                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 561                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 562         },
 563         [MLX5_EXPANSION_IPV4_TCP] = {
 564                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 565                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 566         },
 567         [MLX5_EXPANSION_IPV6] = {
 568                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 569                                                   MLX5_EXPANSION_IPV6_TCP),
 570                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 571                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 572                         ETH_RSS_NONFRAG_IPV6_OTHER,
 573         },
 574         [MLX5_EXPANSION_IPV6_UDP] = {
 575                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 576                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 577         },
 578         [MLX5_EXPANSION_IPV6_TCP] = {
 579                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 580                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 581         },
 582 };
 583
 584 static struct rte_flow_shared_action *
 585 mlx5_shared_action_create(struct rte_eth_dev *dev,
 586                           const struct rte_flow_shared_action_conf *conf,
 587                           const struct rte_flow_action *action,
 588                           struct rte_flow_error *error);
 589 static int mlx5_shared_action_destroy
 590                                 (struct rte_eth_dev *dev,
 591                                  struct rte_flow_shared_action *shared_action,
 592                                  struct rte_flow_error *error);
 593 static int mlx5_shared_action_update
 594                                 (struct rte_eth_dev *dev,
 595                                  struct rte_flow_shared_action *shared_action,
 596                                  const struct rte_flow_action *action,
 597                                  struct rte_flow_error *error);
 598 static int mlx5_shared_action_query
 599                                 (struct rte_eth_dev *dev,
 600                                  const struct rte_flow_shared_action *action,
 601                                  void *data,
 602                                  struct rte_flow_error *error);
 603 static int
 604 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 605                     struct rte_flow_tunnel *app_tunnel,
 606                     struct rte_flow_action **actions,
 607                     uint32_t *num_of_actions,
 608                     struct rte_flow_error *error);
 609 static int
 610 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 611                        struct rte_flow_tunnel *app_tunnel,
 612                        struct rte_flow_item **items,
 613                        uint32_t *num_of_items,
 614                        struct rte_flow_error *error);
 615 static int
 616 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 617                               struct rte_flow_item *pmd_items,
 618                               uint32_t num_items, struct rte_flow_error *err);
 619 static int
 620 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 621                                 struct rte_flow_action *pmd_actions,
 622                                 uint32_t num_actions,
 623                                 struct rte_flow_error *err);
 624 static int
 625 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 626                                   struct rte_mbuf *m,
 627                                   struct rte_flow_restore_info *info,
 628                                   struct rte_flow_error *err);
 629
 630 static const struct rte_flow_ops mlx5_flow_ops = {
 631         .validate = mlx5_flow_validate,
 632         .create = mlx5_flow_create,
 633         .destroy = mlx5_flow_destroy,
 634         .flush = mlx5_flow_flush,
 635         .isolate = mlx5_flow_isolate,
 636         .query = mlx5_flow_query,
 637         .dev_dump = mlx5_flow_dev_dump,
 638         .get_aged_flows = mlx5_flow_get_aged_flows,
 639         .shared_action_create = mlx5_shared_action_create,
 640         .shared_action_destroy = mlx5_shared_action_destroy,
 641         .shared_action_update = mlx5_shared_action_update,
 642         .shared_action_query = mlx5_shared_action_query,
 643         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 644         .tunnel_match = mlx5_flow_tunnel_match,
 645         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 646         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 647         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 648 };
 649
 650 /* Tunnel information. */
 651 struct mlx5_flow_tunnel_info {
 652         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 653         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 654 };
 655
 656 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 657         {
 658                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 659                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 660         },
 661         {
 662                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 663                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 664         },
 665         {
 666                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 667                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 668         },
 669         {
 670                 .tunnel = MLX5_FLOW_LAYER_GRE,
 671                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 672         },
 673         {
 674                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 675                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 676         },
 677         {
 678                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 679                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 680         },
 681         {
 682                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 683                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 684         },
 685         {
 686                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 687                 .ptype = RTE_PTYPE_TUNNEL_IP,
 688         },
 689         {
 690                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 691                 .ptype = RTE_PTYPE_TUNNEL_IP,
 692         },
 693         {
 694                 .tunnel = MLX5_FLOW_LAYER_GTP,
 695                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 696         },
 697 };
 698
 699 /* Key of thread specific flow workspace data. */
 700 static pthread_key_t key_workspace;
 701
 702 /* Thread specific flow workspace data once initialization data. */
 703 static pthread_once_t key_workspace_init;
 704
 705
 706 /**
 707  * Translate tag ID to register.
 708  *
 709  * @param[in] dev
 710  *   Pointer to the Ethernet device structure.
 711  * @param[in] feature
 712  *   The feature that request the register.
 713  * @param[in] id
 714  *   The request register ID.
 715  * @param[out] error
 716  *   Error description in case of any.
 717  *
 718  * @return
 719  *   The request register on success, a negative errno
 720  *   value otherwise and rte_errno is set.
 721  */
 722 int
 723 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 724                      enum mlx5_feature_name feature,
 725                      uint32_t id,
 726                      struct rte_flow_error *error)
 727 {
 728         struct mlx5_priv *priv = dev->data->dev_private;
 729         struct mlx5_dev_config *config = &priv->config;
 730         enum modify_reg start_reg;
 731         bool skip_mtr_reg = false;
 732
 733         switch (feature) {
 734         case MLX5_HAIRPIN_RX:
 735                 return REG_B;
 736         case MLX5_HAIRPIN_TX:
 737                 return REG_A;
 738         case MLX5_METADATA_RX:
 739                 switch (config->dv_xmeta_en) {
 740                 case MLX5_XMETA_MODE_LEGACY:
 741                         return REG_B;
 742                 case MLX5_XMETA_MODE_META16:
 743                         return REG_C_0;
 744                 case MLX5_XMETA_MODE_META32:
 745                         return REG_C_1;
 746                 }
 747                 break;
 748         case MLX5_METADATA_TX:
 749                 return REG_A;
 750         case MLX5_METADATA_FDB:
 751                 switch (config->dv_xmeta_en) {
 752                 case MLX5_XMETA_MODE_LEGACY:
 753                         return REG_NON;
 754                 case MLX5_XMETA_MODE_META16:
 755                         return REG_C_0;
 756                 case MLX5_XMETA_MODE_META32:
 757                         return REG_C_1;
 758                 }
 759                 break;
 760         case MLX5_FLOW_MARK:
 761                 switch (config->dv_xmeta_en) {
 762                 case MLX5_XMETA_MODE_LEGACY:
 763                         return REG_NON;
 764                 case MLX5_XMETA_MODE_META16:
 765                         return REG_C_1;
 766                 case MLX5_XMETA_MODE_META32:
 767                         return REG_C_0;
 768                 }
 769                 break;
 770         case MLX5_MTR_SFX:
 771                 /*
 772                  * If meter color and flow match share one register, flow match
 773                  * should use the meter color register for match.
 774                  */
 775                 if (priv->mtr_reg_share)
 776                         return priv->mtr_color_reg;
 777                 else
 778                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 779                                REG_C_3;
 780         case MLX5_MTR_COLOR:
 781         case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */
 782                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 783                 return priv->mtr_color_reg;
 784         case MLX5_COPY_MARK:
 785                 /*
 786                  * Metadata COPY_MARK register using is in meter suffix sub
 787                  * flow while with meter. It's safe to share the same register.
 788                  */
 789                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 790         case MLX5_APP_TAG:
 791                 /*
 792                  * If meter is enable, it will engage the register for color
 793                  * match and flow match. If meter color match is not using the
 794                  * REG_C_2, need to skip the REG_C_x be used by meter color
 795                  * match.
 796                  * If meter is disable, free to use all available registers.
 797                  */
 798                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 799                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 800                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 801                 if (id > (REG_C_7 - start_reg))
 802                         return rte_flow_error_set(error, EINVAL,
 803                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 804                                                   NULL, "invalid tag id");
 805                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 806                         return rte_flow_error_set(error, ENOTSUP,
 807                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 808                                                   NULL, "unsupported tag id");
 809                 /*
 810                  * This case means meter is using the REG_C_x great than 2.
 811                  * Take care not to conflict with meter color REG_C_x.
 812                  * If the available index REG_C_y >= REG_C_x, skip the
 813                  * color register.
 814                  */
 815                 if (skip_mtr_reg && config->flow_mreg_c
 816                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 817                         if (id >= (REG_C_7 - start_reg))
 818                                 return rte_flow_error_set(error, EINVAL,
 819                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 820                                                         NULL, "invalid tag id");
 821                         if (config->flow_mreg_c
 822                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 823                                 return config->flow_mreg_c
 824                                                [id + 1 + start_reg - REG_C_0];
 825                         return rte_flow_error_set(error, ENOTSUP,
 826                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 827                                                   NULL, "unsupported tag id");
 828                 }
 829                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 830         }
 831         MLX5_ASSERT(false);
 832         return rte_flow_error_set(error, EINVAL,
 833                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 834                                   NULL, "invalid feature name");
 835 }
 836
 837 /**
 838  * Check extensive flow metadata register support.
 839  *
 840  * @param dev
 841  *   Pointer to rte_eth_dev structure.
 842  *
 843  * @return
 844  *   True if device supports extensive flow metadata register, otherwise false.
 845  */
 846 bool
 847 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 848 {
 849         struct mlx5_priv *priv = dev->data->dev_private;
 850         struct mlx5_dev_config *config = &priv->config;
 851
 852         /*
 853          * Having available reg_c can be regarded inclusively as supporting
 854          * extensive flow metadata register, which could mean,
 855          * - metadata register copy action by modify header.
 856          * - 16 modify header actions is supported.
 857          * - reg_c's are preserved across different domain (FDB and NIC) on
 858          *   packet loopback by flow lookup miss.
 859          */
 860         return config->flow_mreg_c[2] != REG_NON;
 861 }
 862
 863 /**
 864  * Verify the @p item specifications (spec, last, mask) are compatible with the
 865  * NIC capabilities.
 866  *
 867  * @param[in] item
 868  *   Item specification.
 869  * @param[in] mask
 870  *   @p item->mask or flow default bit-masks.
 871  * @param[in] nic_mask
 872  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 873  * @param[in] size
 874  *   Bit-masks size in bytes.
 875  * @param[in] range_accepted
 876  *   True if range of values is accepted for specific fields, false otherwise.
 877  * @param[out] error
 878  *   Pointer to error structure.
 879  *
 880  * @return
 881  *   0 on success, a negative errno value otherwise and rte_errno is set.
 882  */
 883 int
 884 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 885                           const uint8_t *mask,
 886                           const uint8_t *nic_mask,
 887                           unsigned int size,
 888                           bool range_accepted,
 889                           struct rte_flow_error *error)
 890 {
 891         unsigned int i;
 892
 893         MLX5_ASSERT(nic_mask);
 894         for (i = 0; i < size; ++i)
 895                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 896                         return rte_flow_error_set(error, ENOTSUP,
 897                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 898                                                   item,
 899                                                   "mask enables non supported"
 900                                                   " bits");
 901         if (!item->spec && (item->mask || item->last))
 902                 return rte_flow_error_set(error, EINVAL,
 903                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 904                                           "mask/last without a spec is not"
 905                                           " supported");
 906         if (item->spec && item->last && !range_accepted) {
 907                 uint8_t spec[size];
 908                 uint8_t last[size];
 909                 unsigned int i;
 910                 int ret;
 911
 912                 for (i = 0; i < size; ++i) {
 913                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 914                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 915                 }
 916                 ret = memcmp(spec, last, size);
 917                 if (ret != 0)
 918                         return rte_flow_error_set(error, EINVAL,
 919                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 920                                                   item,
 921                                                   "range is not valid");
 922         }
 923         return 0;
 924 }
 925
 926 /**
 927  * Adjust the hash fields according to the @p flow information.
 928  *
 929  * @param[in] dev_flow.
 930  *   Pointer to the mlx5_flow.
 931  * @param[in] tunnel
 932  *   1 when the hash field is for a tunnel item.
 933  * @param[in] layer_types
 934  *   ETH_RSS_* types.
 935  * @param[in] hash_fields
 936  *   Item hash fields.
 937  *
 938  * @return
 939  *   The hash fields that should be used.
 940  */
 941 uint64_t
 942 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 943                             int tunnel __rte_unused, uint64_t layer_types,
 944                             uint64_t hash_fields)
 945 {
 946 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 947         int rss_request_inner = rss_desc->level >= 2;
 948
 949         /* Check RSS hash level for tunnel. */
 950         if (tunnel && rss_request_inner)
 951                 hash_fields |= IBV_RX_HASH_INNER;
 952         else if (tunnel || rss_request_inner)
 953                 return 0;
 954 #endif
 955         /* Check if requested layer matches RSS hash fields. */
 956         if (!(rss_desc->types & layer_types))
 957                 return 0;
 958         return hash_fields;
 959 }
 960
 961 /**
 962  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 963  * if several tunnel rules are used on this queue, the tunnel ptype will be
 964  * cleared.
 965  *
 966  * @param rxq_ctrl
 967  *   Rx queue to update.
 968  */
 969 static void
 970 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 971 {
 972         unsigned int i;
 973         uint32_t tunnel_ptype = 0;
 974
 975         /* Look up for the ptype to use. */
 976         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 977                 if (!rxq_ctrl->flow_tunnels_n[i])
 978                         continue;
 979                 if (!tunnel_ptype) {
 980                         tunnel_ptype = tunnels_info[i].ptype;
 981                 } else {
 982                         tunnel_ptype = 0;
 983                         break;
 984                 }
 985         }
 986         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 987 }
 988
 989 /**
 990  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 991  * flow.
 992  *
 993  * @param[in] dev
 994  *   Pointer to the Ethernet device structure.
 995  * @param[in] dev_handle
 996  *   Pointer to device flow handle structure.
 997  */
 998 static void
 999 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1000                        struct mlx5_flow_handle *dev_handle)
1001 {
1002         struct mlx5_priv *priv = dev->data->dev_private;
1003         const int mark = dev_handle->mark;
1004         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1005         struct mlx5_hrxq *hrxq;
1006         unsigned int i;
1007
1008         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1009                 return;
1010         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1011                               dev_handle->rix_hrxq);
1012         if (!hrxq)
1013                 return;
1014         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1015                 int idx = hrxq->ind_table->queues[i];
1016                 struct mlx5_rxq_ctrl *rxq_ctrl =
1017                         container_of((*priv->rxqs)[idx],
1018                                      struct mlx5_rxq_ctrl, rxq);
1019
1020                 /*
1021                  * To support metadata register copy on Tx loopback,
1022                  * this must be always enabled (metadata may arive
1023                  * from other port - not from local flows only.
1024                  */
1025                 if (priv->config.dv_flow_en &&
1026                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1027                     mlx5_flow_ext_mreg_supported(dev)) {
1028                         rxq_ctrl->rxq.mark = 1;
1029                         rxq_ctrl->flow_mark_n = 1;
1030                 } else if (mark) {
1031                         rxq_ctrl->rxq.mark = 1;
1032                         rxq_ctrl->flow_mark_n++;
1033                 }
1034                 if (tunnel) {
1035                         unsigned int j;
1036
1037                         /* Increase the counter matching the flow. */
1038                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1039                                 if ((tunnels_info[j].tunnel &
1040                                      dev_handle->layers) ==
1041                                     tunnels_info[j].tunnel) {
1042                                         rxq_ctrl->flow_tunnels_n[j]++;
1043                                         break;
1044                                 }
1045                         }
1046                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1047                 }
1048         }
1049 }
1050
1051 /**
1052  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1053  *
1054  * @param[in] dev
1055  *   Pointer to the Ethernet device structure.
1056  * @param[in] flow
1057  *   Pointer to flow structure.
1058  */
1059 static void
1060 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1061 {
1062         struct mlx5_priv *priv = dev->data->dev_private;
1063         uint32_t handle_idx;
1064         struct mlx5_flow_handle *dev_handle;
1065
1066         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1067                        handle_idx, dev_handle, next)
1068                 flow_drv_rxq_flags_set(dev, dev_handle);
1069 }
1070
1071 /**
1072  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1073  * device flow if no other flow uses it with the same kind of request.
1074  *
1075  * @param dev
1076  *   Pointer to Ethernet device.
1077  * @param[in] dev_handle
1078  *   Pointer to the device flow handle structure.
1079  */
1080 static void
1081 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1082                         struct mlx5_flow_handle *dev_handle)
1083 {
1084         struct mlx5_priv *priv = dev->data->dev_private;
1085         const int mark = dev_handle->mark;
1086         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1087         struct mlx5_hrxq *hrxq;
1088         unsigned int i;
1089
1090         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1091                 return;
1092         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1093                               dev_handle->rix_hrxq);
1094         if (!hrxq)
1095                 return;
1096         MLX5_ASSERT(dev->data->dev_started);
1097         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1098                 int idx = hrxq->ind_table->queues[i];
1099                 struct mlx5_rxq_ctrl *rxq_ctrl =
1100                         container_of((*priv->rxqs)[idx],
1101                                      struct mlx5_rxq_ctrl, rxq);
1102
1103                 if (priv->config.dv_flow_en &&
1104                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1105                     mlx5_flow_ext_mreg_supported(dev)) {
1106                         rxq_ctrl->rxq.mark = 1;
1107                         rxq_ctrl->flow_mark_n = 1;
1108                 } else if (mark) {
1109                         rxq_ctrl->flow_mark_n--;
1110                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1111                 }
1112                 if (tunnel) {
1113                         unsigned int j;
1114
1115                         /* Decrease the counter matching the flow. */
1116                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1117                                 if ((tunnels_info[j].tunnel &
1118                                      dev_handle->layers) ==
1119                                     tunnels_info[j].tunnel) {
1120                                         rxq_ctrl->flow_tunnels_n[j]--;
1121                                         break;
1122                                 }
1123                         }
1124                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1125                 }
1126         }
1127 }
1128
1129 /**
1130  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1131  * @p flow if no other flow uses it with the same kind of request.
1132  *
1133  * @param dev
1134  *   Pointer to Ethernet device.
1135  * @param[in] flow
1136  *   Pointer to the flow.
1137  */
1138 static void
1139 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1140 {
1141         struct mlx5_priv *priv = dev->data->dev_private;
1142         uint32_t handle_idx;
1143         struct mlx5_flow_handle *dev_handle;
1144
1145         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1146                        handle_idx, dev_handle, next)
1147                 flow_drv_rxq_flags_trim(dev, dev_handle);
1148 }
1149
1150 /**
1151  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1152  *
1153  * @param dev
1154  *   Pointer to Ethernet device.
1155  */
1156 static void
1157 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1158 {
1159         struct mlx5_priv *priv = dev->data->dev_private;
1160         unsigned int i;
1161
1162         for (i = 0; i != priv->rxqs_n; ++i) {
1163                 struct mlx5_rxq_ctrl *rxq_ctrl;
1164                 unsigned int j;
1165
1166                 if (!(*priv->rxqs)[i])
1167                         continue;
1168                 rxq_ctrl = container_of((*priv->rxqs)[i],
1169                                         struct mlx5_rxq_ctrl, rxq);
1170                 rxq_ctrl->flow_mark_n = 0;
1171                 rxq_ctrl->rxq.mark = 0;
1172                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1173                         rxq_ctrl->flow_tunnels_n[j] = 0;
1174                 rxq_ctrl->rxq.tunnel = 0;
1175         }
1176 }
1177
1178 /**
1179  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1180  *
1181  * @param[in] dev
1182  *   Pointer to the Ethernet device structure.
1183  */
1184 void
1185 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1186 {
1187         struct mlx5_priv *priv = dev->data->dev_private;
1188         struct mlx5_rxq_data *data;
1189         unsigned int i;
1190
1191         for (i = 0; i != priv->rxqs_n; ++i) {
1192                 if (!(*priv->rxqs)[i])
1193                         continue;
1194                 data = (*priv->rxqs)[i];
1195                 if (!rte_flow_dynf_metadata_avail()) {
1196                         data->dynf_meta = 0;
1197                         data->flow_meta_mask = 0;
1198                         data->flow_meta_offset = -1;
1199                 } else {
1200                         data->dynf_meta = 1;
1201                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1202                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1203                 }
1204         }
1205 }
1206
1207 /*
1208  * return a pointer to the desired action in the list of actions.
1209  *
1210  * @param[in] actions
1211  *   The list of actions to search the action in.
1212  * @param[in] action
1213  *   The action to find.
1214  *
1215  * @return
1216  *   Pointer to the action in the list, if found. NULL otherwise.
1217  */
1218 const struct rte_flow_action *
1219 mlx5_flow_find_action(const struct rte_flow_action *actions,
1220                       enum rte_flow_action_type action)
1221 {
1222         if (actions == NULL)
1223                 return NULL;
1224         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1225                 if (actions->type == action)
1226                         return actions;
1227         return NULL;
1228 }
1229
1230 /*
1231  * Validate the flag action.
1232  *
1233  * @param[in] action_flags
1234  *   Bit-fields that holds the actions detected until now.
1235  * @param[in] attr
1236  *   Attributes of flow that includes this action.
1237  * @param[out] error
1238  *   Pointer to error structure.
1239  *
1240  * @return
1241  *   0 on success, a negative errno value otherwise and rte_errno is set.
1242  */
1243 int
1244 mlx5_flow_validate_action_flag(uint64_t action_flags,
1245                                const struct rte_flow_attr *attr,
1246                                struct rte_flow_error *error)
1247 {
1248         if (action_flags & MLX5_FLOW_ACTION_MARK)
1249                 return rte_flow_error_set(error, EINVAL,
1250                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1251                                           "can't mark and flag in same flow");
1252         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1253                 return rte_flow_error_set(error, EINVAL,
1254                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1255                                           "can't have 2 flag"
1256                                           " actions in same flow");
1257         if (attr->egress)
1258                 return rte_flow_error_set(error, ENOTSUP,
1259                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1260                                           "flag action not supported for "
1261                                           "egress");
1262         return 0;
1263 }
1264
1265 /*
1266  * Validate the mark action.
1267  *
1268  * @param[in] action
1269  *   Pointer to the queue action.
1270  * @param[in] action_flags
1271  *   Bit-fields that holds the actions detected until now.
1272  * @param[in] attr
1273  *   Attributes of flow that includes this action.
1274  * @param[out] error
1275  *   Pointer to error structure.
1276  *
1277  * @return
1278  *   0 on success, a negative errno value otherwise and rte_errno is set.
1279  */
1280 int
1281 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1282                                uint64_t action_flags,
1283                                const struct rte_flow_attr *attr,
1284                                struct rte_flow_error *error)
1285 {
1286         const struct rte_flow_action_mark *mark = action->conf;
1287
1288         if (!mark)
1289                 return rte_flow_error_set(error, EINVAL,
1290                                           RTE_FLOW_ERROR_TYPE_ACTION,
1291                                           action,
1292                                           "configuration cannot be null");
1293         if (mark->id >= MLX5_FLOW_MARK_MAX)
1294                 return rte_flow_error_set(error, EINVAL,
1295                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1296                                           &mark->id,
1297                                           "mark id must in 0 <= id < "
1298                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1299         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1300                 return rte_flow_error_set(error, EINVAL,
1301                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1302                                           "can't flag and mark in same flow");
1303         if (action_flags & MLX5_FLOW_ACTION_MARK)
1304                 return rte_flow_error_set(error, EINVAL,
1305                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1306                                           "can't have 2 mark actions in same"
1307                                           " flow");
1308         if (attr->egress)
1309                 return rte_flow_error_set(error, ENOTSUP,
1310                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1311                                           "mark action not supported for "
1312                                           "egress");
1313         return 0;
1314 }
1315
1316 /*
1317  * Validate the drop action.
1318  *
1319  * @param[in] action_flags
1320  *   Bit-fields that holds the actions detected until now.
1321  * @param[in] attr
1322  *   Attributes of flow that includes this action.
1323  * @param[out] error
1324  *   Pointer to error structure.
1325  *
1326  * @return
1327  *   0 on success, a negative errno value otherwise and rte_errno is set.
1328  */
1329 int
1330 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1331                                const struct rte_flow_attr *attr,
1332                                struct rte_flow_error *error)
1333 {
1334         if (attr->egress)
1335                 return rte_flow_error_set(error, ENOTSUP,
1336                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1337                                           "drop action not supported for "
1338                                           "egress");
1339         return 0;
1340 }
1341
1342 /*
1343  * Validate the queue action.
1344  *
1345  * @param[in] action
1346  *   Pointer to the queue action.
1347  * @param[in] action_flags
1348  *   Bit-fields that holds the actions detected until now.
1349  * @param[in] dev
1350  *   Pointer to the Ethernet device structure.
1351  * @param[in] attr
1352  *   Attributes of flow that includes this action.
1353  * @param[out] error
1354  *   Pointer to error structure.
1355  *
1356  * @return
1357  *   0 on success, a negative errno value otherwise and rte_errno is set.
1358  */
1359 int
1360 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1361                                 uint64_t action_flags,
1362                                 struct rte_eth_dev *dev,
1363                                 const struct rte_flow_attr *attr,
1364                                 struct rte_flow_error *error)
1365 {
1366         struct mlx5_priv *priv = dev->data->dev_private;
1367         const struct rte_flow_action_queue *queue = action->conf;
1368
1369         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1370                 return rte_flow_error_set(error, EINVAL,
1371                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1372                                           "can't have 2 fate actions in"
1373                                           " same flow");
1374         if (!priv->rxqs_n)
1375                 return rte_flow_error_set(error, EINVAL,
1376                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1377                                           NULL, "No Rx queues configured");
1378         if (queue->index >= priv->rxqs_n)
1379                 return rte_flow_error_set(error, EINVAL,
1380                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1381                                           &queue->index,
1382                                           "queue index out of range");
1383         if (!(*priv->rxqs)[queue->index])
1384                 return rte_flow_error_set(error, EINVAL,
1385                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1386                                           &queue->index,
1387                                           "queue is not configured");
1388         if (attr->egress)
1389                 return rte_flow_error_set(error, ENOTSUP,
1390                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1391                                           "queue action not supported for "
1392                                           "egress");
1393         return 0;
1394 }
1395
1396 /*
1397  * Validate the rss action.
1398  *
1399  * @param[in] dev
1400  *   Pointer to the Ethernet device structure.
1401  * @param[in] action
1402  *   Pointer to the queue action.
1403  * @param[out] error
1404  *   Pointer to error structure.
1405  *
1406  * @return
1407  *   0 on success, a negative errno value otherwise and rte_errno is set.
1408  */
1409 int
1410 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1411                          const struct rte_flow_action *action,
1412                          struct rte_flow_error *error)
1413 {
1414         struct mlx5_priv *priv = dev->data->dev_private;
1415         const struct rte_flow_action_rss *rss = action->conf;
1416         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1417         unsigned int i;
1418
1419         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1420             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1421                 return rte_flow_error_set(error, ENOTSUP,
1422                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1423                                           &rss->func,
1424                                           "RSS hash function not supported");
1425 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1426         if (rss->level > 2)
1427 #else
1428         if (rss->level > 1)
1429 #endif
1430                 return rte_flow_error_set(error, ENOTSUP,
1431                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1432                                           &rss->level,
1433                                           "tunnel RSS is not supported");
1434         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1435         if (rss->key_len == 0 && rss->key != NULL)
1436                 return rte_flow_error_set(error, ENOTSUP,
1437                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1438                                           &rss->key_len,
1439                                           "RSS hash key length 0");
1440         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1441                 return rte_flow_error_set(error, ENOTSUP,
1442                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1443                                           &rss->key_len,
1444                                           "RSS hash key too small");
1445         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1446                 return rte_flow_error_set(error, ENOTSUP,
1447                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1448                                           &rss->key_len,
1449                                           "RSS hash key too large");
1450         if (rss->queue_num > priv->config.ind_table_max_size)
1451                 return rte_flow_error_set(error, ENOTSUP,
1452                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1453                                           &rss->queue_num,
1454                                           "number of queues too large");
1455         if (rss->types & MLX5_RSS_HF_MASK)
1456                 return rte_flow_error_set(error, ENOTSUP,
1457                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1458                                           &rss->types,
1459                                           "some RSS protocols are not"
1460                                           " supported");
1461         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1462             !(rss->types & ETH_RSS_IP))
1463                 return rte_flow_error_set(error, EINVAL,
1464                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1465                                           "L3 partial RSS requested but L3 RSS"
1466                                           " type not specified");
1467         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1468             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1469                 return rte_flow_error_set(error, EINVAL,
1470                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1471                                           "L4 partial RSS requested but L4 RSS"
1472                                           " type not specified");
1473         if (!priv->rxqs_n)
1474                 return rte_flow_error_set(error, EINVAL,
1475                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1476                                           NULL, "No Rx queues configured");
1477         if (!rss->queue_num)
1478                 return rte_flow_error_set(error, EINVAL,
1479                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1480                                           NULL, "No queues configured");
1481         for (i = 0; i != rss->queue_num; ++i) {
1482                 struct mlx5_rxq_ctrl *rxq_ctrl;
1483
1484                 if (rss->queue[i] >= priv->rxqs_n)
1485                         return rte_flow_error_set
1486                                 (error, EINVAL,
1487                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1488                                  &rss->queue[i], "queue index out of range");
1489                 if (!(*priv->rxqs)[rss->queue[i]])
1490                         return rte_flow_error_set
1491                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1492                                  &rss->queue[i], "queue is not configured");
1493                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1494                                         struct mlx5_rxq_ctrl, rxq);
1495                 if (i == 0)
1496                         rxq_type = rxq_ctrl->type;
1497                 if (rxq_type != rxq_ctrl->type)
1498                         return rte_flow_error_set
1499                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1500                                  &rss->queue[i],
1501                                  "combining hairpin and regular RSS queues is not supported");
1502         }
1503         return 0;
1504 }
1505
1506 /*
1507  * Validate the rss action.
1508  *
1509  * @param[in] action
1510  *   Pointer to the queue action.
1511  * @param[in] action_flags
1512  *   Bit-fields that holds the actions detected until now.
1513  * @param[in] dev
1514  *   Pointer to the Ethernet device structure.
1515  * @param[in] attr
1516  *   Attributes of flow that includes this action.
1517  * @param[in] item_flags
1518  *   Items that were detected.
1519  * @param[out] error
1520  *   Pointer to error structure.
1521  *
1522  * @return
1523  *   0 on success, a negative errno value otherwise and rte_errno is set.
1524  */
1525 int
1526 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1527                               uint64_t action_flags,
1528                               struct rte_eth_dev *dev,
1529                               const struct rte_flow_attr *attr,
1530                               uint64_t item_flags,
1531                               struct rte_flow_error *error)
1532 {
1533         const struct rte_flow_action_rss *rss = action->conf;
1534         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1535         int ret;
1536
1537         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1538                 return rte_flow_error_set(error, EINVAL,
1539                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1540                                           "can't have 2 fate actions"
1541                                           " in same flow");
1542         ret = mlx5_validate_action_rss(dev, action, error);
1543         if (ret)
1544                 return ret;
1545         if (attr->egress)
1546                 return rte_flow_error_set(error, ENOTSUP,
1547                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1548                                           "rss action not supported for "
1549                                           "egress");
1550         if (rss->level > 1 && !tunnel)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1553                                           "inner RSS is not supported for "
1554                                           "non-tunnel flows");
1555         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1556             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1557                 return rte_flow_error_set(error, EINVAL,
1558                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1559                                           "RSS on eCPRI is not supported now");
1560         }
1561         return 0;
1562 }
1563
1564 /*
1565  * Validate the default miss action.
1566  *
1567  * @param[in] action_flags
1568  *   Bit-fields that holds the actions detected until now.
1569  * @param[out] error
1570  *   Pointer to error structure.
1571  *
1572  * @return
1573  *   0 on success, a negative errno value otherwise and rte_errno is set.
1574  */
1575 int
1576 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1577                                 const struct rte_flow_attr *attr,
1578                                 struct rte_flow_error *error)
1579 {
1580         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1581                 return rte_flow_error_set(error, EINVAL,
1582                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1583                                           "can't have 2 fate actions in"
1584                                           " same flow");
1585         if (attr->egress)
1586                 return rte_flow_error_set(error, ENOTSUP,
1587                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1588                                           "default miss action not supported "
1589                                           "for egress");
1590         if (attr->group)
1591                 return rte_flow_error_set(error, ENOTSUP,
1592                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1593                                           "only group 0 is supported");
1594         if (attr->transfer)
1595                 return rte_flow_error_set(error, ENOTSUP,
1596                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1597                                           NULL, "transfer is not supported");
1598         return 0;
1599 }
1600
1601 /*
1602  * Validate the count action.
1603  *
1604  * @param[in] dev
1605  *   Pointer to the Ethernet device structure.
1606  * @param[in] attr
1607  *   Attributes of flow that includes this action.
1608  * @param[out] error
1609  *   Pointer to error structure.
1610  *
1611  * @return
1612  *   0 on success, a negative errno value otherwise and rte_errno is set.
1613  */
1614 int
1615 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1616                                 const struct rte_flow_attr *attr,
1617                                 struct rte_flow_error *error)
1618 {
1619         if (attr->egress)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1622                                           "count action not supported for "
1623                                           "egress");
1624         return 0;
1625 }
1626
1627 /**
1628  * Verify the @p attributes will be correctly understood by the NIC and store
1629  * them in the @p flow if everything is correct.
1630  *
1631  * @param[in] dev
1632  *   Pointer to the Ethernet device structure.
1633  * @param[in] attributes
1634  *   Pointer to flow attributes
1635  * @param[out] error
1636  *   Pointer to error structure.
1637  *
1638  * @return
1639  *   0 on success, a negative errno value otherwise and rte_errno is set.
1640  */
1641 int
1642 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1643                               const struct rte_flow_attr *attributes,
1644                               struct rte_flow_error *error)
1645 {
1646         struct mlx5_priv *priv = dev->data->dev_private;
1647         uint32_t priority_max = priv->config.flow_prio - 1;
1648
1649         if (attributes->group)
1650                 return rte_flow_error_set(error, ENOTSUP,
1651                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1652                                           NULL, "groups is not supported");
1653         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1654             attributes->priority >= priority_max)
1655                 return rte_flow_error_set(error, ENOTSUP,
1656                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1657                                           NULL, "priority out of range");
1658         if (attributes->egress)
1659                 return rte_flow_error_set(error, ENOTSUP,
1660                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1661                                           "egress is not supported");
1662         if (attributes->transfer && !priv->config.dv_esw_en)
1663                 return rte_flow_error_set(error, ENOTSUP,
1664                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1665                                           NULL, "transfer is not supported");
1666         if (!attributes->ingress)
1667                 return rte_flow_error_set(error, EINVAL,
1668                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1669                                           NULL,
1670                                           "ingress attribute is mandatory");
1671         return 0;
1672 }
1673
1674 /**
1675  * Validate ICMP6 item.
1676  *
1677  * @param[in] item
1678  *   Item specification.
1679  * @param[in] item_flags
1680  *   Bit-fields that holds the items detected until now.
1681  * @param[in] ext_vlan_sup
1682  *   Whether extended VLAN features are supported or not.
1683  * @param[out] error
1684  *   Pointer to error structure.
1685  *
1686  * @return
1687  *   0 on success, a negative errno value otherwise and rte_errno is set.
1688  */
1689 int
1690 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1691                                uint64_t item_flags,
1692                                uint8_t target_protocol,
1693                                struct rte_flow_error *error)
1694 {
1695         const struct rte_flow_item_icmp6 *mask = item->mask;
1696         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1697         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1698                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1699         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1700                                       MLX5_FLOW_LAYER_OUTER_L4;
1701         int ret;
1702
1703         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1704                 return rte_flow_error_set(error, EINVAL,
1705                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1706                                           "protocol filtering not compatible"
1707                                           " with ICMP6 layer");
1708         if (!(item_flags & l3m))
1709                 return rte_flow_error_set(error, EINVAL,
1710                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1711                                           "IPv6 is mandatory to filter on"
1712                                           " ICMP6");
1713         if (item_flags & l4m)
1714                 return rte_flow_error_set(error, EINVAL,
1715                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1716                                           "multiple L4 layers not supported");
1717         if (!mask)
1718                 mask = &rte_flow_item_icmp6_mask;
1719         ret = mlx5_flow_item_acceptable
1720                 (item, (const uint8_t *)mask,
1721                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1722                  sizeof(struct rte_flow_item_icmp6),
1723                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1724         if (ret < 0)
1725                 return ret;
1726         return 0;
1727 }
1728
1729 /**
1730  * Validate ICMP item.
1731  *
1732  * @param[in] item
1733  *   Item specification.
1734  * @param[in] item_flags
1735  *   Bit-fields that holds the items detected until now.
1736  * @param[out] error
1737  *   Pointer to error structure.
1738  *
1739  * @return
1740  *   0 on success, a negative errno value otherwise and rte_errno is set.
1741  */
1742 int
1743 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1744                              uint64_t item_flags,
1745                              uint8_t target_protocol,
1746                              struct rte_flow_error *error)
1747 {
1748         const struct rte_flow_item_icmp *mask = item->mask;
1749         const struct rte_flow_item_icmp nic_mask = {
1750                 .hdr.icmp_type = 0xff,
1751                 .hdr.icmp_code = 0xff,
1752                 .hdr.icmp_ident = RTE_BE16(0xffff),
1753                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1754         };
1755         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1756         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1757                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1758         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1759                                       MLX5_FLOW_LAYER_OUTER_L4;
1760         int ret;
1761
1762         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1763                 return rte_flow_error_set(error, EINVAL,
1764                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1765                                           "protocol filtering not compatible"
1766                                           " with ICMP layer");
1767         if (!(item_flags & l3m))
1768                 return rte_flow_error_set(error, EINVAL,
1769                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1770                                           "IPv4 is mandatory to filter"
1771                                           " on ICMP");
1772         if (item_flags & l4m)
1773                 return rte_flow_error_set(error, EINVAL,
1774                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1775                                           "multiple L4 layers not supported");
1776         if (!mask)
1777                 mask = &nic_mask;
1778         ret = mlx5_flow_item_acceptable
1779                 (item, (const uint8_t *)mask,
1780                  (const uint8_t *)&nic_mask,
1781                  sizeof(struct rte_flow_item_icmp),
1782                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1783         if (ret < 0)
1784                 return ret;
1785         return 0;
1786 }
1787
1788 /**
1789  * Validate Ethernet item.
1790  *
1791  * @param[in] item
1792  *   Item specification.
1793  * @param[in] item_flags
1794  *   Bit-fields that holds the items detected until now.
1795  * @param[out] error
1796  *   Pointer to error structure.
1797  *
1798  * @return
1799  *   0 on success, a negative errno value otherwise and rte_errno is set.
1800  */
1801 int
1802 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1803                             uint64_t item_flags, bool ext_vlan_sup,
1804                             struct rte_flow_error *error)
1805 {
1806         const struct rte_flow_item_eth *mask = item->mask;
1807         const struct rte_flow_item_eth nic_mask = {
1808                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1809                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1810                 .type = RTE_BE16(0xffff),
1811                 .has_vlan = ext_vlan_sup ? 1 : 0,
1812         };
1813         int ret;
1814         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1815         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1816                                        MLX5_FLOW_LAYER_OUTER_L2;
1817
1818         if (item_flags & ethm)
1819                 return rte_flow_error_set(error, ENOTSUP,
1820                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1821                                           "multiple L2 layers not supported");
1822         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1823             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1824                 return rte_flow_error_set(error, EINVAL,
1825                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1826                                           "L2 layer should not follow "
1827                                           "L3 layers");
1828         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1829             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1830                 return rte_flow_error_set(error, EINVAL,
1831                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1832                                           "L2 layer should not follow VLAN");
1833         if (!mask)
1834                 mask = &rte_flow_item_eth_mask;
1835         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1836                                         (const uint8_t *)&nic_mask,
1837                                         sizeof(struct rte_flow_item_eth),
1838                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1839         return ret;
1840 }
1841
1842 /**
1843  * Validate VLAN item.
1844  *
1845  * @param[in] item
1846  *   Item specification.
1847  * @param[in] item_flags
1848  *   Bit-fields that holds the items detected until now.
1849  * @param[in] dev
1850  *   Ethernet device flow is being created on.
1851  * @param[out] error
1852  *   Pointer to error structure.
1853  *
1854  * @return
1855  *   0 on success, a negative errno value otherwise and rte_errno is set.
1856  */
1857 int
1858 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1859                              uint64_t item_flags,
1860                              struct rte_eth_dev *dev,
1861                              struct rte_flow_error *error)
1862 {
1863         const struct rte_flow_item_vlan *spec = item->spec;
1864         const struct rte_flow_item_vlan *mask = item->mask;
1865         const struct rte_flow_item_vlan nic_mask = {
1866                 .tci = RTE_BE16(UINT16_MAX),
1867                 .inner_type = RTE_BE16(UINT16_MAX),
1868         };
1869         uint16_t vlan_tag = 0;
1870         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1871         int ret;
1872         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1873                                         MLX5_FLOW_LAYER_INNER_L4) :
1874                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1875                                         MLX5_FLOW_LAYER_OUTER_L4);
1876         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1877                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1878
1879         if (item_flags & vlanm)
1880                 return rte_flow_error_set(error, EINVAL,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1882                                           "multiple VLAN layers not supported");
1883         else if ((item_flags & l34m) != 0)
1884                 return rte_flow_error_set(error, EINVAL,
1885                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1886                                           "VLAN cannot follow L3/L4 layer");
1887         if (!mask)
1888                 mask = &rte_flow_item_vlan_mask;
1889         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1890                                         (const uint8_t *)&nic_mask,
1891                                         sizeof(struct rte_flow_item_vlan),
1892                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1893         if (ret)
1894                 return ret;
1895         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1896                 struct mlx5_priv *priv = dev->data->dev_private;
1897
1898                 if (priv->vmwa_context) {
1899                         /*
1900                          * Non-NULL context means we have a virtual machine
1901                          * and SR-IOV enabled, we have to create VLAN interface
1902                          * to make hypervisor to setup E-Switch vport
1903                          * context correctly. We avoid creating the multiple
1904                          * VLAN interfaces, so we cannot support VLAN tag mask.
1905                          */
1906                         return rte_flow_error_set(error, EINVAL,
1907                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1908                                                   item,
1909                                                   "VLAN tag mask is not"
1910                                                   " supported in virtual"
1911                                                   " environment");
1912                 }
1913         }
1914         if (spec) {
1915                 vlan_tag = spec->tci;
1916                 vlan_tag &= mask->tci;
1917         }
1918         /*
1919          * From verbs perspective an empty VLAN is equivalent
1920          * to a packet without VLAN layer.
1921          */
1922         if (!vlan_tag)
1923                 return rte_flow_error_set(error, EINVAL,
1924                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1925                                           item->spec,
1926                                           "VLAN cannot be empty");
1927         return 0;
1928 }
1929
1930 /**
1931  * Validate IPV4 item.
1932  *
1933  * @param[in] item
1934  *   Item specification.
1935  * @param[in] item_flags
1936  *   Bit-fields that holds the items detected until now.
1937  * @param[in] last_item
1938  *   Previous validated item in the pattern items.
1939  * @param[in] ether_type
1940  *   Type in the ethernet layer header (including dot1q).
1941  * @param[in] acc_mask
1942  *   Acceptable mask, if NULL default internal default mask
1943  *   will be used to check whether item fields are supported.
1944  * @param[in] range_accepted
1945  *   True if range of values is accepted for specific fields, false otherwise.
1946  * @param[out] error
1947  *   Pointer to error structure.
1948  *
1949  * @return
1950  *   0 on success, a negative errno value otherwise and rte_errno is set.
1951  */
1952 int
1953 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1954                              uint64_t item_flags,
1955                              uint64_t last_item,
1956                              uint16_t ether_type,
1957                              const struct rte_flow_item_ipv4 *acc_mask,
1958                              bool range_accepted,
1959                              struct rte_flow_error *error)
1960 {
1961         const struct rte_flow_item_ipv4 *mask = item->mask;
1962         const struct rte_flow_item_ipv4 *spec = item->spec;
1963         const struct rte_flow_item_ipv4 nic_mask = {
1964                 .hdr = {
1965                         .src_addr = RTE_BE32(0xffffffff),
1966                         .dst_addr = RTE_BE32(0xffffffff),
1967                         .type_of_service = 0xff,
1968                         .next_proto_id = 0xff,
1969                 },
1970         };
1971         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1972         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1973                                       MLX5_FLOW_LAYER_OUTER_L3;
1974         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1975                                       MLX5_FLOW_LAYER_OUTER_L4;
1976         int ret;
1977         uint8_t next_proto = 0xFF;
1978         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1979                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1980                                   MLX5_FLOW_LAYER_INNER_VLAN);
1981
1982         if ((last_item & l2_vlan) && ether_type &&
1983             ether_type != RTE_ETHER_TYPE_IPV4)
1984                 return rte_flow_error_set(error, EINVAL,
1985                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1986                                           "IPv4 cannot follow L2/VLAN layer "
1987                                           "which ether type is not IPv4");
1988         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1989                 if (mask && spec)
1990                         next_proto = mask->hdr.next_proto_id &
1991                                      spec->hdr.next_proto_id;
1992                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1993                         return rte_flow_error_set(error, EINVAL,
1994                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1995                                                   item,
1996                                                   "multiple tunnel "
1997                                                   "not supported");
1998         }
1999         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2000                 return rte_flow_error_set(error, EINVAL,
2001                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2002                                           "wrong tunnel type - IPv6 specified "
2003                                           "but IPv4 item provided");
2004         if (item_flags & l3m)
2005                 return rte_flow_error_set(error, ENOTSUP,
2006                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2007                                           "multiple L3 layers not supported");
2008         else if (item_flags & l4m)
2009                 return rte_flow_error_set(error, EINVAL,
2010                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2011                                           "L3 cannot follow an L4 layer.");
2012         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2013                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2014                 return rte_flow_error_set(error, EINVAL,
2015                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2016                                           "L3 cannot follow an NVGRE layer.");
2017         if (!mask)
2018                 mask = &rte_flow_item_ipv4_mask;
2019         else if (mask->hdr.next_proto_id != 0 &&
2020                  mask->hdr.next_proto_id != 0xff)
2021                 return rte_flow_error_set(error, EINVAL,
2022                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2023                                           "partial mask is not supported"
2024                                           " for protocol");
2025         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2026                                         acc_mask ? (const uint8_t *)acc_mask
2027                                                  : (const uint8_t *)&nic_mask,
2028                                         sizeof(struct rte_flow_item_ipv4),
2029                                         range_accepted, error);
2030         if (ret < 0)
2031                 return ret;
2032         return 0;
2033 }
2034
2035 /**
2036  * Validate IPV6 item.
2037  *
2038  * @param[in] item
2039  *   Item specification.
2040  * @param[in] item_flags
2041  *   Bit-fields that holds the items detected until now.
2042  * @param[in] last_item
2043  *   Previous validated item in the pattern items.
2044  * @param[in] ether_type
2045  *   Type in the ethernet layer header (including dot1q).
2046  * @param[in] acc_mask
2047  *   Acceptable mask, if NULL default internal default mask
2048  *   will be used to check whether item fields are supported.
2049  * @param[out] error
2050  *   Pointer to error structure.
2051  *
2052  * @return
2053  *   0 on success, a negative errno value otherwise and rte_errno is set.
2054  */
2055 int
2056 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2057                              uint64_t item_flags,
2058                              uint64_t last_item,
2059                              uint16_t ether_type,
2060                              const struct rte_flow_item_ipv6 *acc_mask,
2061                              struct rte_flow_error *error)
2062 {
2063         const struct rte_flow_item_ipv6 *mask = item->mask;
2064         const struct rte_flow_item_ipv6 *spec = item->spec;
2065         const struct rte_flow_item_ipv6 nic_mask = {
2066                 .hdr = {
2067                         .src_addr =
2068                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2069                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2070                         .dst_addr =
2071                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2072                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2073                         .vtc_flow = RTE_BE32(0xffffffff),
2074                         .proto = 0xff,
2075                 },
2076         };
2077         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2078         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2079                                       MLX5_FLOW_LAYER_OUTER_L3;
2080         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2081                                       MLX5_FLOW_LAYER_OUTER_L4;
2082         int ret;
2083         uint8_t next_proto = 0xFF;
2084         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2085                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2086                                   MLX5_FLOW_LAYER_INNER_VLAN);
2087
2088         if ((last_item & l2_vlan) && ether_type &&
2089             ether_type != RTE_ETHER_TYPE_IPV6)
2090                 return rte_flow_error_set(error, EINVAL,
2091                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2092                                           "IPv6 cannot follow L2/VLAN layer "
2093                                           "which ether type is not IPv6");
2094         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2095                 next_proto = spec->hdr.proto;
2096         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2097                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2098                         return rte_flow_error_set(error, EINVAL,
2099                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2100                                                   item,
2101                                                   "multiple tunnel "
2102                                                   "not supported");
2103         }
2104         if (next_proto == IPPROTO_HOPOPTS  ||
2105             next_proto == IPPROTO_ROUTING  ||
2106             next_proto == IPPROTO_FRAGMENT ||
2107             next_proto == IPPROTO_ESP      ||
2108             next_proto == IPPROTO_AH       ||
2109             next_proto == IPPROTO_DSTOPTS)
2110                 return rte_flow_error_set(error, EINVAL,
2111                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2112                                           "IPv6 proto (next header) should "
2113                                           "not be set as extension header");
2114         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2115                 return rte_flow_error_set(error, EINVAL,
2116                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2117                                           "wrong tunnel type - IPv4 specified "
2118                                           "but IPv6 item provided");
2119         if (item_flags & l3m)
2120                 return rte_flow_error_set(error, ENOTSUP,
2121                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2122                                           "multiple L3 layers not supported");
2123         else if (item_flags & l4m)
2124                 return rte_flow_error_set(error, EINVAL,
2125                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2126                                           "L3 cannot follow an L4 layer.");
2127         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2128                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2129                 return rte_flow_error_set(error, EINVAL,
2130                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2131                                           "L3 cannot follow an NVGRE layer.");
2132         if (!mask)
2133                 mask = &rte_flow_item_ipv6_mask;
2134         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2135                                         acc_mask ? (const uint8_t *)acc_mask
2136                                                  : (const uint8_t *)&nic_mask,
2137                                         sizeof(struct rte_flow_item_ipv6),
2138                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2139         if (ret < 0)
2140                 return ret;
2141         return 0;
2142 }
2143
2144 /**
2145  * Validate UDP item.
2146  *
2147  * @param[in] item
2148  *   Item specification.
2149  * @param[in] item_flags
2150  *   Bit-fields that holds the items detected until now.
2151  * @param[in] target_protocol
2152  *   The next protocol in the previous item.
2153  * @param[in] flow_mask
2154  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2155  * @param[out] error
2156  *   Pointer to error structure.
2157  *
2158  * @return
2159  *   0 on success, a negative errno value otherwise and rte_errno is set.
2160  */
2161 int
2162 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2163                             uint64_t item_flags,
2164                             uint8_t target_protocol,
2165                             struct rte_flow_error *error)
2166 {
2167         const struct rte_flow_item_udp *mask = item->mask;
2168         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2169         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2170                                       MLX5_FLOW_LAYER_OUTER_L3;
2171         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2172                                       MLX5_FLOW_LAYER_OUTER_L4;
2173         int ret;
2174
2175         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2176                 return rte_flow_error_set(error, EINVAL,
2177                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2178                                           "protocol filtering not compatible"
2179                                           " with UDP layer");
2180         if (!(item_flags & l3m))
2181                 return rte_flow_error_set(error, EINVAL,
2182                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2183                                           "L3 is mandatory to filter on L4");
2184         if (item_flags & l4m)
2185                 return rte_flow_error_set(error, EINVAL,
2186                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2187                                           "multiple L4 layers not supported");
2188         if (!mask)
2189                 mask = &rte_flow_item_udp_mask;
2190         ret = mlx5_flow_item_acceptable
2191                 (item, (const uint8_t *)mask,
2192                  (const uint8_t *)&rte_flow_item_udp_mask,
2193                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2194                  error);
2195         if (ret < 0)
2196                 return ret;
2197         return 0;
2198 }
2199
2200 /**
2201  * Validate TCP item.
2202  *
2203  * @param[in] item
2204  *   Item specification.
2205  * @param[in] item_flags
2206  *   Bit-fields that holds the items detected until now.
2207  * @param[in] target_protocol
2208  *   The next protocol in the previous item.
2209  * @param[out] error
2210  *   Pointer to error structure.
2211  *
2212  * @return
2213  *   0 on success, a negative errno value otherwise and rte_errno is set.
2214  */
2215 int
2216 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2217                             uint64_t item_flags,
2218                             uint8_t target_protocol,
2219                             const struct rte_flow_item_tcp *flow_mask,
2220                             struct rte_flow_error *error)
2221 {
2222         const struct rte_flow_item_tcp *mask = item->mask;
2223         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2224         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2225                                       MLX5_FLOW_LAYER_OUTER_L3;
2226         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2227                                       MLX5_FLOW_LAYER_OUTER_L4;
2228         int ret;
2229
2230         MLX5_ASSERT(flow_mask);
2231         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2232                 return rte_flow_error_set(error, EINVAL,
2233                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2234                                           "protocol filtering not compatible"
2235                                           " with TCP layer");
2236         if (!(item_flags & l3m))
2237                 return rte_flow_error_set(error, EINVAL,
2238                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2239                                           "L3 is mandatory to filter on L4");
2240         if (item_flags & l4m)
2241                 return rte_flow_error_set(error, EINVAL,
2242                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2243                                           "multiple L4 layers not supported");
2244         if (!mask)
2245                 mask = &rte_flow_item_tcp_mask;
2246         ret = mlx5_flow_item_acceptable
2247                 (item, (const uint8_t *)mask,
2248                  (const uint8_t *)flow_mask,
2249                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2250                  error);
2251         if (ret < 0)
2252                 return ret;
2253         return 0;
2254 }
2255
2256 /**
2257  * Validate VXLAN item.
2258  *
2259  * @param[in] item
2260  *   Item specification.
2261  * @param[in] item_flags
2262  *   Bit-fields that holds the items detected until now.
2263  * @param[in] target_protocol
2264  *   The next protocol in the previous item.
2265  * @param[out] error
2266  *   Pointer to error structure.
2267  *
2268  * @return
2269  *   0 on success, a negative errno value otherwise and rte_errno is set.
2270  */
2271 int
2272 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2273                               uint64_t item_flags,
2274                               struct rte_flow_error *error)
2275 {
2276         const struct rte_flow_item_vxlan *spec = item->spec;
2277         const struct rte_flow_item_vxlan *mask = item->mask;
2278         int ret;
2279         union vni {
2280                 uint32_t vlan_id;
2281                 uint8_t vni[4];
2282         } id = { .vlan_id = 0, };
2283
2284
2285         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2286                 return rte_flow_error_set(error, ENOTSUP,
2287                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2288                                           "multiple tunnel layers not"
2289                                           " supported");
2290         /*
2291          * Verify only UDPv4 is present as defined in
2292          * https://tools.ietf.org/html/rfc7348
2293          */
2294         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2295                 return rte_flow_error_set(error, EINVAL,
2296                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2297                                           "no outer UDP layer found");
2298         if (!mask)
2299                 mask = &rte_flow_item_vxlan_mask;
2300         ret = mlx5_flow_item_acceptable
2301                 (item, (const uint8_t *)mask,
2302                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2303                  sizeof(struct rte_flow_item_vxlan),
2304                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2305         if (ret < 0)
2306                 return ret;
2307         if (spec) {
2308                 memcpy(&id.vni[1], spec->vni, 3);
2309                 memcpy(&id.vni[1], mask->vni, 3);
2310         }
2311         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2312                 return rte_flow_error_set(error, ENOTSUP,
2313                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2314                                           "VXLAN tunnel must be fully defined");
2315         return 0;
2316 }
2317
2318 /**
2319  * Validate VXLAN_GPE item.
2320  *
2321  * @param[in] item
2322  *   Item specification.
2323  * @param[in] item_flags
2324  *   Bit-fields that holds the items detected until now.
2325  * @param[in] priv
2326  *   Pointer to the private data structure.
2327  * @param[in] target_protocol
2328  *   The next protocol in the previous item.
2329  * @param[out] error
2330  *   Pointer to error structure.
2331  *
2332  * @return
2333  *   0 on success, a negative errno value otherwise and rte_errno is set.
2334  */
2335 int
2336 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2337                                   uint64_t item_flags,
2338                                   struct rte_eth_dev *dev,
2339                                   struct rte_flow_error *error)
2340 {
2341         struct mlx5_priv *priv = dev->data->dev_private;
2342         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2343         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2344         int ret;
2345         union vni {
2346                 uint32_t vlan_id;
2347                 uint8_t vni[4];
2348         } id = { .vlan_id = 0, };
2349
2350         if (!priv->config.l3_vxlan_en)
2351                 return rte_flow_error_set(error, ENOTSUP,
2352                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2353                                           "L3 VXLAN is not enabled by device"
2354                                           " parameter and/or not configured in"
2355                                           " firmware");
2356         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2359                                           "multiple tunnel layers not"
2360                                           " supported");
2361         /*
2362          * Verify only UDPv4 is present as defined in
2363          * https://tools.ietf.org/html/rfc7348
2364          */
2365         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2366                 return rte_flow_error_set(error, EINVAL,
2367                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2368                                           "no outer UDP layer found");
2369         if (!mask)
2370                 mask = &rte_flow_item_vxlan_gpe_mask;
2371         ret = mlx5_flow_item_acceptable
2372                 (item, (const uint8_t *)mask,
2373                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2374                  sizeof(struct rte_flow_item_vxlan_gpe),
2375                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2376         if (ret < 0)
2377                 return ret;
2378         if (spec) {
2379                 if (spec->protocol)
2380                         return rte_flow_error_set(error, ENOTSUP,
2381                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2382                                                   item,
2383                                                   "VxLAN-GPE protocol"
2384                                                   " not supported");
2385                 memcpy(&id.vni[1], spec->vni, 3);
2386                 memcpy(&id.vni[1], mask->vni, 3);
2387         }
2388         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2389                 return rte_flow_error_set(error, ENOTSUP,
2390                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2391                                           "VXLAN-GPE tunnel must be fully"
2392                                           " defined");
2393         return 0;
2394 }
2395 /**
2396  * Validate GRE Key item.
2397  *
2398  * @param[in] item
2399  *   Item specification.
2400  * @param[in] item_flags
2401  *   Bit flags to mark detected items.
2402  * @param[in] gre_item
2403  *   Pointer to gre_item
2404  * @param[out] error
2405  *   Pointer to error structure.
2406  *
2407  * @return
2408  *   0 on success, a negative errno value otherwise and rte_errno is set.
2409  */
2410 int
2411 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2412                                 uint64_t item_flags,
2413                                 const struct rte_flow_item *gre_item,
2414                                 struct rte_flow_error *error)
2415 {
2416         const rte_be32_t *mask = item->mask;
2417         int ret = 0;
2418         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2419         const struct rte_flow_item_gre *gre_spec;
2420         const struct rte_flow_item_gre *gre_mask;
2421
2422         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2423                 return rte_flow_error_set(error, ENOTSUP,
2424                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2425                                           "Multiple GRE key not support");
2426         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2427                 return rte_flow_error_set(error, ENOTSUP,
2428                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2429                                           "No preceding GRE header");
2430         if (item_flags & MLX5_FLOW_LAYER_INNER)
2431                 return rte_flow_error_set(error, ENOTSUP,
2432                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2433                                           "GRE key following a wrong item");
2434         gre_mask = gre_item->mask;
2435         if (!gre_mask)
2436                 gre_mask = &rte_flow_item_gre_mask;
2437         gre_spec = gre_item->spec;
2438         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2439                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2442                                           "Key bit must be on");
2443
2444         if (!mask)
2445                 mask = &gre_key_default_mask;
2446         ret = mlx5_flow_item_acceptable
2447                 (item, (const uint8_t *)mask,
2448                  (const uint8_t *)&gre_key_default_mask,
2449                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2450         return ret;
2451 }
2452
2453 /**
2454  * Validate GRE item.
2455  *
2456  * @param[in] item
2457  *   Item specification.
2458  * @param[in] item_flags
2459  *   Bit flags to mark detected items.
2460  * @param[in] target_protocol
2461  *   The next protocol in the previous item.
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 int
2469 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2470                             uint64_t item_flags,
2471                             uint8_t target_protocol,
2472                             struct rte_flow_error *error)
2473 {
2474         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2475         const struct rte_flow_item_gre *mask = item->mask;
2476         int ret;
2477         const struct rte_flow_item_gre nic_mask = {
2478                 .c_rsvd0_ver = RTE_BE16(0xB000),
2479                 .protocol = RTE_BE16(UINT16_MAX),
2480         };
2481
2482         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2483                 return rte_flow_error_set(error, EINVAL,
2484                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2485                                           "protocol filtering not compatible"
2486                                           " with this GRE layer");
2487         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2488                 return rte_flow_error_set(error, ENOTSUP,
2489                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2490                                           "multiple tunnel layers not"
2491                                           " supported");
2492         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2493                 return rte_flow_error_set(error, ENOTSUP,
2494                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2495                                           "L3 Layer is missing");
2496         if (!mask)
2497                 mask = &rte_flow_item_gre_mask;
2498         ret = mlx5_flow_item_acceptable
2499                 (item, (const uint8_t *)mask,
2500                  (const uint8_t *)&nic_mask,
2501                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2502                  error);
2503         if (ret < 0)
2504                 return ret;
2505 #ifndef HAVE_MLX5DV_DR
2506 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2507         if (spec && (spec->protocol & mask->protocol))
2508                 return rte_flow_error_set(error, ENOTSUP,
2509                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2510                                           "without MPLS support the"
2511                                           " specification cannot be used for"
2512                                           " filtering");
2513 #endif
2514 #endif
2515         return 0;
2516 }
2517
2518 /**
2519  * Validate Geneve item.
2520  *
2521  * @param[in] item
2522  *   Item specification.
2523  * @param[in] itemFlags
2524  *   Bit-fields that holds the items detected until now.
2525  * @param[in] enPriv
2526  *   Pointer to the private data structure.
2527  * @param[out] error
2528  *   Pointer to error structure.
2529  *
2530  * @return
2531  *   0 on success, a negative errno value otherwise and rte_errno is set.
2532  */
2533
2534 int
2535 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2536                                uint64_t item_flags,
2537                                struct rte_eth_dev *dev,
2538                                struct rte_flow_error *error)
2539 {
2540         struct mlx5_priv *priv = dev->data->dev_private;
2541         const struct rte_flow_item_geneve *spec = item->spec;
2542         const struct rte_flow_item_geneve *mask = item->mask;
2543         int ret;
2544         uint16_t gbhdr;
2545         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2546                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2547         const struct rte_flow_item_geneve nic_mask = {
2548                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2549                 .vni = "\xff\xff\xff",
2550                 .protocol = RTE_BE16(UINT16_MAX),
2551         };
2552
2553         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2554                 return rte_flow_error_set(error, ENOTSUP,
2555                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2556                                           "L3 Geneve is not enabled by device"
2557                                           " parameter and/or not configured in"
2558                                           " firmware");
2559         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2560                 return rte_flow_error_set(error, ENOTSUP,
2561                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2562                                           "multiple tunnel layers not"
2563                                           " supported");
2564         /*
2565          * Verify only UDPv4 is present as defined in
2566          * https://tools.ietf.org/html/rfc7348
2567          */
2568         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2569                 return rte_flow_error_set(error, EINVAL,
2570                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2571                                           "no outer UDP layer found");
2572         if (!mask)
2573                 mask = &rte_flow_item_geneve_mask;
2574         ret = mlx5_flow_item_acceptable
2575                                   (item, (const uint8_t *)mask,
2576                                    (const uint8_t *)&nic_mask,
2577                                    sizeof(struct rte_flow_item_geneve),
2578                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2579         if (ret)
2580                 return ret;
2581         if (spec) {
2582                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2583                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2584                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2585                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2586                         return rte_flow_error_set(error, ENOTSUP,
2587                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2588                                                   item,
2589                                                   "Geneve protocol unsupported"
2590                                                   " fields are being used");
2591                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2592                         return rte_flow_error_set
2593                                         (error, ENOTSUP,
2594                                          RTE_FLOW_ERROR_TYPE_ITEM,
2595                                          item,
2596                                          "Unsupported Geneve options length");
2597         }
2598         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2599                 return rte_flow_error_set
2600                                     (error, ENOTSUP,
2601                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2602                                      "Geneve tunnel must be fully defined");
2603         return 0;
2604 }
2605
2606 /**
2607  * Validate MPLS item.
2608  *
2609  * @param[in] dev
2610  *   Pointer to the rte_eth_dev structure.
2611  * @param[in] item
2612  *   Item specification.
2613  * @param[in] item_flags
2614  *   Bit-fields that holds the items detected until now.
2615  * @param[in] prev_layer
2616  *   The protocol layer indicated in previous item.
2617  * @param[out] error
2618  *   Pointer to error structure.
2619  *
2620  * @return
2621  *   0 on success, a negative errno value otherwise and rte_errno is set.
2622  */
2623 int
2624 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2625                              const struct rte_flow_item *item __rte_unused,
2626                              uint64_t item_flags __rte_unused,
2627                              uint64_t prev_layer __rte_unused,
2628                              struct rte_flow_error *error)
2629 {
2630 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2631         const struct rte_flow_item_mpls *mask = item->mask;
2632         struct mlx5_priv *priv = dev->data->dev_private;
2633         int ret;
2634
2635         if (!priv->config.mpls_en)
2636                 return rte_flow_error_set(error, ENOTSUP,
2637                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2638                                           "MPLS not supported or"
2639                                           " disabled in firmware"
2640                                           " configuration.");
2641         /* MPLS over IP, UDP, GRE is allowed */
2642         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2643                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2644                             MLX5_FLOW_LAYER_GRE |
2645                             MLX5_FLOW_LAYER_GRE_KEY)))
2646                 return rte_flow_error_set(error, EINVAL,
2647                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2648                                           "protocol filtering not compatible"
2649                                           " with MPLS layer");
2650         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2651         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2652             !(item_flags & MLX5_FLOW_LAYER_GRE))
2653                 return rte_flow_error_set(error, ENOTSUP,
2654                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2655                                           "multiple tunnel layers not"
2656                                           " supported");
2657         if (!mask)
2658                 mask = &rte_flow_item_mpls_mask;
2659         ret = mlx5_flow_item_acceptable
2660                 (item, (const uint8_t *)mask,
2661                  (const uint8_t *)&rte_flow_item_mpls_mask,
2662                  sizeof(struct rte_flow_item_mpls),
2663                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2664         if (ret < 0)
2665                 return ret;
2666         return 0;
2667 #else
2668         return rte_flow_error_set(error, ENOTSUP,
2669                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2670                                   "MPLS is not supported by Verbs, please"
2671                                   " update.");
2672 #endif
2673 }
2674
2675 /**
2676  * Validate NVGRE item.
2677  *
2678  * @param[in] item
2679  *   Item specification.
2680  * @param[in] item_flags
2681  *   Bit flags to mark detected items.
2682  * @param[in] target_protocol
2683  *   The next protocol in the previous item.
2684  * @param[out] error
2685  *   Pointer to error structure.
2686  *
2687  * @return
2688  *   0 on success, a negative errno value otherwise and rte_errno is set.
2689  */
2690 int
2691 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2692                               uint64_t item_flags,
2693                               uint8_t target_protocol,
2694                               struct rte_flow_error *error)
2695 {
2696         const struct rte_flow_item_nvgre *mask = item->mask;
2697         int ret;
2698
2699         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2700                 return rte_flow_error_set(error, EINVAL,
2701                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2702                                           "protocol filtering not compatible"
2703                                           " with this GRE layer");
2704         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2705                 return rte_flow_error_set(error, ENOTSUP,
2706                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2707                                           "multiple tunnel layers not"
2708                                           " supported");
2709         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2710                 return rte_flow_error_set(error, ENOTSUP,
2711                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2712                                           "L3 Layer is missing");
2713         if (!mask)
2714                 mask = &rte_flow_item_nvgre_mask;
2715         ret = mlx5_flow_item_acceptable
2716                 (item, (const uint8_t *)mask,
2717                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2718                  sizeof(struct rte_flow_item_nvgre),
2719                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2720         if (ret < 0)
2721                 return ret;
2722         return 0;
2723 }
2724
2725 /**
2726  * Validate eCPRI item.
2727  *
2728  * @param[in] item
2729  *   Item specification.
2730  * @param[in] item_flags
2731  *   Bit-fields that holds the items detected until now.
2732  * @param[in] last_item
2733  *   Previous validated item in the pattern items.
2734  * @param[in] ether_type
2735  *   Type in the ethernet layer header (including dot1q).
2736  * @param[in] acc_mask
2737  *   Acceptable mask, if NULL default internal default mask
2738  *   will be used to check whether item fields are supported.
2739  * @param[out] error
2740  *   Pointer to error structure.
2741  *
2742  * @return
2743  *   0 on success, a negative errno value otherwise and rte_errno is set.
2744  */
2745 int
2746 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2747                               uint64_t item_flags,
2748                               uint64_t last_item,
2749                               uint16_t ether_type,
2750                               const struct rte_flow_item_ecpri *acc_mask,
2751                               struct rte_flow_error *error)
2752 {
2753         const struct rte_flow_item_ecpri *mask = item->mask;
2754         const struct rte_flow_item_ecpri nic_mask = {
2755                 .hdr = {
2756                         .common = {
2757                                 .u32 =
2758                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2759                                         .type = 0xFF,
2760                                         }).u32),
2761                         },
2762                         .dummy[0] = 0xFFFFFFFF,
2763                 },
2764         };
2765         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2766                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2767         struct rte_flow_item_ecpri mask_lo;
2768
2769         if (!(last_item & outer_l2_vlan) &&
2770             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
2771                 return rte_flow_error_set(error, EINVAL,
2772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2773                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
2774         if ((last_item & outer_l2_vlan) && ether_type &&
2775             ether_type != RTE_ETHER_TYPE_ECPRI)
2776                 return rte_flow_error_set(error, EINVAL,
2777                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2778                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
2779         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2780                 return rte_flow_error_set(error, EINVAL,
2781                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2782                                           "eCPRI with tunnel is not supported right now");
2783         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2784                 return rte_flow_error_set(error, ENOTSUP,
2785                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2786                                           "multiple L3 layers not supported");
2787         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2788                 return rte_flow_error_set(error, EINVAL,
2789                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2790                                           "eCPRI cannot coexist with a TCP layer");
2791         /* In specification, eCPRI could be over UDP layer. */
2792         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2793                 return rte_flow_error_set(error, EINVAL,
2794                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2795                                           "eCPRI over UDP layer is not yet supported right now");
2796         /* Mask for type field in common header could be zero. */
2797         if (!mask)
2798                 mask = &rte_flow_item_ecpri_mask;
2799         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2800         /* Input mask is in big-endian format. */
2801         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2802                 return rte_flow_error_set(error, EINVAL,
2803                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2804                                           "partial mask is not supported for protocol");
2805         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2806                 return rte_flow_error_set(error, EINVAL,
2807                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2808                                           "message header mask must be after a type mask");
2809         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2810                                          acc_mask ? (const uint8_t *)acc_mask
2811                                                   : (const uint8_t *)&nic_mask,
2812                                          sizeof(struct rte_flow_item_ecpri),
2813                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2814 }
2815
2816 /**
2817  * Release resource related QUEUE/RSS action split.
2818  *
2819  * @param dev
2820  *   Pointer to Ethernet device.
2821  * @param flow
2822  *   Flow to release id's from.
2823  */
2824 static void
2825 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2826                              struct rte_flow *flow)
2827 {
2828         struct mlx5_priv *priv = dev->data->dev_private;
2829         uint32_t handle_idx;
2830         struct mlx5_flow_handle *dev_handle;
2831
2832         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2833                        handle_idx, dev_handle, next)
2834                 if (dev_handle->split_flow_id)
2835                         mlx5_ipool_free(priv->sh->ipool
2836                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
2837                                         dev_handle->split_flow_id);
2838 }
2839
2840 static int
2841 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2842                    const struct rte_flow_attr *attr __rte_unused,
2843                    const struct rte_flow_item items[] __rte_unused,
2844                    const struct rte_flow_action actions[] __rte_unused,
2845                    bool external __rte_unused,
2846                    int hairpin __rte_unused,
2847                    struct rte_flow_error *error)
2848 {
2849         return rte_flow_error_set(error, ENOTSUP,
2850                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2851 }
2852
2853 static struct mlx5_flow *
2854 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2855                   const struct rte_flow_attr *attr __rte_unused,
2856                   const struct rte_flow_item items[] __rte_unused,
2857                   const struct rte_flow_action actions[] __rte_unused,
2858                   struct rte_flow_error *error)
2859 {
2860         rte_flow_error_set(error, ENOTSUP,
2861                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2862         return NULL;
2863 }
2864
2865 static int
2866 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2867                     struct mlx5_flow *dev_flow __rte_unused,
2868                     const struct rte_flow_attr *attr __rte_unused,
2869                     const struct rte_flow_item items[] __rte_unused,
2870                     const struct rte_flow_action actions[] __rte_unused,
2871                     struct rte_flow_error *error)
2872 {
2873         return rte_flow_error_set(error, ENOTSUP,
2874                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2875 }
2876
2877 static int
2878 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2879                 struct rte_flow *flow __rte_unused,
2880                 struct rte_flow_error *error)
2881 {
2882         return rte_flow_error_set(error, ENOTSUP,
2883                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2884 }
2885
2886 static void
2887 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2888                  struct rte_flow *flow __rte_unused)
2889 {
2890 }
2891
2892 static void
2893 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2894                   struct rte_flow *flow __rte_unused)
2895 {
2896 }
2897
2898 static int
2899 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2900                 struct rte_flow *flow __rte_unused,
2901                 const struct rte_flow_action *actions __rte_unused,
2902                 void *data __rte_unused,
2903                 struct rte_flow_error *error)
2904 {
2905         return rte_flow_error_set(error, ENOTSUP,
2906                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2907 }
2908
2909 static int
2910 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
2911                       uint32_t domains __rte_unused,
2912                       uint32_t flags __rte_unused)
2913 {
2914         return 0;
2915 }
2916
2917 /* Void driver to protect from null pointer reference. */
2918 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2919         .validate = flow_null_validate,
2920         .prepare = flow_null_prepare,
2921         .translate = flow_null_translate,
2922         .apply = flow_null_apply,
2923         .remove = flow_null_remove,
2924         .destroy = flow_null_destroy,
2925         .query = flow_null_query,
2926         .sync_domain = flow_null_sync_domain,
2927 };
2928
2929 /**
2930  * Select flow driver type according to flow attributes and device
2931  * configuration.
2932  *
2933  * @param[in] dev
2934  *   Pointer to the dev structure.
2935  * @param[in] attr
2936  *   Pointer to the flow attributes.
2937  *
2938  * @return
2939  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2940  */
2941 static enum mlx5_flow_drv_type
2942 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2943 {
2944         struct mlx5_priv *priv = dev->data->dev_private;
2945         /* The OS can determine first a specific flow type (DV, VERBS) */
2946         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2947
2948         if (type != MLX5_FLOW_TYPE_MAX)
2949                 return type;
2950         /* If no OS specific type - continue with DV/VERBS selection */
2951         if (attr->transfer && priv->config.dv_esw_en)
2952                 type = MLX5_FLOW_TYPE_DV;
2953         if (!attr->transfer)
2954                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2955                                                  MLX5_FLOW_TYPE_VERBS;
2956         return type;
2957 }
2958
2959 #define flow_get_drv_ops(type) flow_drv_ops[type]
2960
2961 /**
2962  * Flow driver validation API. This abstracts calling driver specific functions.
2963  * The type of flow driver is determined according to flow attributes.
2964  *
2965  * @param[in] dev
2966  *   Pointer to the dev structure.
2967  * @param[in] attr
2968  *   Pointer to the flow attributes.
2969  * @param[in] items
2970  *   Pointer to the list of items.
2971  * @param[in] actions
2972  *   Pointer to the list of actions.
2973  * @param[in] external
2974  *   This flow rule is created by request external to PMD.
2975  * @param[in] hairpin
2976  *   Number of hairpin TX actions, 0 means classic flow.
2977  * @param[out] error
2978  *   Pointer to the error structure.
2979  *
2980  * @return
2981  *   0 on success, a negative errno value otherwise and rte_errno is set.
2982  */
2983 static inline int
2984 flow_drv_validate(struct rte_eth_dev *dev,
2985                   const struct rte_flow_attr *attr,
2986                   const struct rte_flow_item items[],
2987                   const struct rte_flow_action actions[],
2988                   bool external, int hairpin, struct rte_flow_error *error)
2989 {
2990         const struct mlx5_flow_driver_ops *fops;
2991         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2992
2993         fops = flow_get_drv_ops(type);
2994         return fops->validate(dev, attr, items, actions, external,
2995                               hairpin, error);
2996 }
2997
2998 /**
2999  * Flow driver preparation API. This abstracts calling driver specific
3000  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3001  * calculates the size of memory required for device flow, allocates the memory,
3002  * initializes the device flow and returns the pointer.
3003  *
3004  * @note
3005  *   This function initializes device flow structure such as dv or verbs in
3006  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3007  *   rest. For example, adding returning device flow to flow->dev_flow list and
3008  *   setting backward reference to the flow should be done out of this function.
3009  *   layers field is not filled either.
3010  *
3011  * @param[in] dev
3012  *   Pointer to the dev structure.
3013  * @param[in] attr
3014  *   Pointer to the flow attributes.
3015  * @param[in] items
3016  *   Pointer to the list of items.
3017  * @param[in] actions
3018  *   Pointer to the list of actions.
3019  * @param[in] flow_idx
3020  *   This memory pool index to the flow.
3021  * @param[out] error
3022  *   Pointer to the error structure.
3023  *
3024  * @return
3025  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3026  */
3027 static inline struct mlx5_flow *
3028 flow_drv_prepare(struct rte_eth_dev *dev,
3029                  const struct rte_flow *flow,
3030                  const struct rte_flow_attr *attr,
3031                  const struct rte_flow_item items[],
3032                  const struct rte_flow_action actions[],
3033                  uint32_t flow_idx,
3034                  struct rte_flow_error *error)
3035 {
3036         const struct mlx5_flow_driver_ops *fops;
3037         enum mlx5_flow_drv_type type = flow->drv_type;
3038         struct mlx5_flow *mlx5_flow = NULL;
3039
3040         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3041         fops = flow_get_drv_ops(type);
3042         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3043         if (mlx5_flow)
3044                 mlx5_flow->flow_idx = flow_idx;
3045         return mlx5_flow;
3046 }
3047
3048 /**
3049  * Flow driver translation API. This abstracts calling driver specific
3050  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3051  * translates a generic flow into a driver flow. flow_drv_prepare() must
3052  * precede.
3053  *
3054  * @note
3055  *   dev_flow->layers could be filled as a result of parsing during translation
3056  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3057  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3058  *   flow->actions could be overwritten even though all the expanded dev_flows
3059  *   have the same actions.
3060  *
3061  * @param[in] dev
3062  *   Pointer to the rte dev structure.
3063  * @param[in, out] dev_flow
3064  *   Pointer to the mlx5 flow.
3065  * @param[in] attr
3066  *   Pointer to the flow attributes.
3067  * @param[in] items
3068  *   Pointer to the list of items.
3069  * @param[in] actions
3070  *   Pointer to the list of actions.
3071  * @param[out] error
3072  *   Pointer to the error structure.
3073  *
3074  * @return
3075  *   0 on success, a negative errno value otherwise and rte_errno is set.
3076  */
3077 static inline int
3078 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3079                    const struct rte_flow_attr *attr,
3080                    const struct rte_flow_item items[],
3081                    const struct rte_flow_action actions[],
3082                    struct rte_flow_error *error)
3083 {
3084         const struct mlx5_flow_driver_ops *fops;
3085         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3086
3087         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3088         fops = flow_get_drv_ops(type);
3089         return fops->translate(dev, dev_flow, attr, items, actions, error);
3090 }
3091
3092 /**
3093  * Flow driver apply API. This abstracts calling driver specific functions.
3094  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3095  * translated driver flows on to device. flow_drv_translate() must precede.
3096  *
3097  * @param[in] dev
3098  *   Pointer to Ethernet device structure.
3099  * @param[in, out] flow
3100  *   Pointer to flow structure.
3101  * @param[out] error
3102  *   Pointer to error structure.
3103  *
3104  * @return
3105  *   0 on success, a negative errno value otherwise and rte_errno is set.
3106  */
3107 static inline int
3108 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3109                struct rte_flow_error *error)
3110 {
3111         const struct mlx5_flow_driver_ops *fops;
3112         enum mlx5_flow_drv_type type = flow->drv_type;
3113
3114         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3115         fops = flow_get_drv_ops(type);
3116         return fops->apply(dev, flow, error);
3117 }
3118
3119 /**
3120  * Flow driver destroy API. This abstracts calling driver specific functions.
3121  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3122  * on device and releases resources of the flow.
3123  *
3124  * @param[in] dev
3125  *   Pointer to Ethernet device.
3126  * @param[in, out] flow
3127  *   Pointer to flow structure.
3128  */
3129 static inline void
3130 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3131 {
3132         const struct mlx5_flow_driver_ops *fops;
3133         enum mlx5_flow_drv_type type = flow->drv_type;
3134
3135         flow_mreg_split_qrss_release(dev, flow);
3136         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3137         fops = flow_get_drv_ops(type);
3138         fops->destroy(dev, flow);
3139 }
3140
3141 /**
3142  * Get RSS action from the action list.
3143  *
3144  * @param[in] actions
3145  *   Pointer to the list of actions.
3146  *
3147  * @return
3148  *   Pointer to the RSS action if exist, else return NULL.
3149  */
3150 static const struct rte_flow_action_rss*
3151 flow_get_rss_action(const struct rte_flow_action actions[])
3152 {
3153         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3154                 switch (actions->type) {
3155                 case RTE_FLOW_ACTION_TYPE_RSS:
3156                         return (const struct rte_flow_action_rss *)
3157                                actions->conf;
3158                 default:
3159                         break;
3160                 }
3161         }
3162         return NULL;
3163 }
3164
3165 /**
3166  * Get ASO age action by index.
3167  *
3168  * @param[in] dev
3169  *   Pointer to the Ethernet device structure.
3170  * @param[in] age_idx
3171  *   Index to the ASO age action.
3172  *
3173  * @return
3174  *   The specified ASO age action.
3175  */
3176 struct mlx5_aso_age_action*
3177 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3178 {
3179         uint16_t pool_idx = age_idx & UINT16_MAX;
3180         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3181         struct mlx5_priv *priv = dev->data->dev_private;
3182         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3183         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3184
3185         return &pool->actions[offset - 1];
3186 }
3187
3188 /* maps shared action to translated non shared in some actions array */
3189 struct mlx5_translated_shared_action {
3190         struct rte_flow_shared_action *action; /**< Shared action */
3191         int index; /**< Index in related array of rte_flow_action */
3192 };
3193
3194 /**
3195  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3196  * non shared action if translation possible.
3197  * This functionality used to run same execution path for both shared & non
3198  * shared actions on flow create. All necessary preparations for shared
3199  * action handling should be preformed on *shared* actions list returned
3200  * from this call.
3201  *
3202  * @param[in] dev
3203  *   Pointer to Ethernet device.
3204  * @param[in] actions
3205  *   List of actions to translate.
3206  * @param[out] shared
3207  *   List to store translated shared actions.
3208  * @param[in, out] shared_n
3209  *   Size of *shared* array. On return should be updated with number of shared
3210  *   actions retrieved from the *actions* list.
3211  * @param[out] translated_actions
3212  *   List of actions where all shared actions were translated to non shared
3213  *   if possible. NULL if no translation took place.
3214  * @param[out] error
3215  *   Pointer to the error structure.
3216  *
3217  * @return
3218  *   0 on success, a negative errno value otherwise and rte_errno is set.
3219  */
3220 static int
3221 flow_shared_actions_translate(struct rte_eth_dev *dev,
3222                               const struct rte_flow_action actions[],
3223                               struct mlx5_translated_shared_action *shared,
3224                               int *shared_n,
3225                               struct rte_flow_action **translated_actions,
3226                               struct rte_flow_error *error)
3227 {
3228         struct mlx5_priv *priv = dev->data->dev_private;
3229         struct rte_flow_action *translated = NULL;
3230         size_t actions_size;
3231         int n;
3232         int copied_n = 0;
3233         struct mlx5_translated_shared_action *shared_end = NULL;
3234
3235         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3236                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3237                         continue;
3238                 if (copied_n == *shared_n) {
3239                         return rte_flow_error_set
3240                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3241                                  NULL, "too many shared actions");
3242                 }
3243                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3244                            sizeof(actions[n].conf));
3245                 shared[copied_n].index = n;
3246                 copied_n++;
3247         }
3248         n++;
3249         *shared_n = copied_n;
3250         if (!copied_n)
3251                 return 0;
3252         actions_size = sizeof(struct rte_flow_action) * n;
3253         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3254         if (!translated) {
3255                 rte_errno = ENOMEM;
3256                 return -ENOMEM;
3257         }
3258         memcpy(translated, actions, actions_size);
3259         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3260                 struct mlx5_shared_action_rss *shared_rss;
3261                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3262                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3263                 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET)
3264                                                                            - 1);
3265
3266                 switch (type) {
3267                 case MLX5_SHARED_ACTION_TYPE_RSS:
3268                         shared_rss = mlx5_ipool_get
3269                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3270                         translated[shared->index].type =
3271                                 RTE_FLOW_ACTION_TYPE_RSS;
3272                         translated[shared->index].conf =
3273                                 &shared_rss->origin;
3274                         break;
3275                 case MLX5_SHARED_ACTION_TYPE_AGE:
3276                         if (priv->sh->flow_hit_aso_en) {
3277                                 translated[shared->index].type =
3278                                         (enum rte_flow_action_type)
3279                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3280                                 translated[shared->index].conf =
3281                                                          (void *)(uintptr_t)idx;
3282                                 break;
3283                         }
3284                         /* Fall-through */
3285                 default:
3286                         mlx5_free(translated);
3287                         return rte_flow_error_set
3288                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3289                                  NULL, "invalid shared action type");
3290                 }
3291         }
3292         *translated_actions = translated;
3293         return 0;
3294 }
3295
3296 /**
3297  * Get Shared RSS action from the action list.
3298  *
3299  * @param[in] dev
3300  *   Pointer to Ethernet device.
3301  * @param[in] shared
3302  *   Pointer to the list of actions.
3303  * @param[in] shared_n
3304  *   Actions list length.
3305  *
3306  * @return
3307  *   The MLX5 RSS action ID if exists, otherwise return 0.
3308  */
3309 static uint32_t
3310 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3311                            struct mlx5_translated_shared_action *shared,
3312                            int shared_n)
3313 {
3314         struct mlx5_translated_shared_action *shared_end;
3315         struct mlx5_priv *priv = dev->data->dev_private;
3316         struct mlx5_shared_action_rss *shared_rss;
3317
3318
3319         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3320                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3321                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3322                 uint32_t idx = act_idx &
3323                                    ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1);
3324                 switch (type) {
3325                 case MLX5_SHARED_ACTION_TYPE_RSS:
3326                         shared_rss = mlx5_ipool_get
3327                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3328                                                                            idx);
3329                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3330                                            __ATOMIC_RELAXED);
3331                         return idx;
3332                 default:
3333                         break;
3334                 }
3335         }
3336         return 0;
3337 }
3338
3339 static unsigned int
3340 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3341 {
3342         const struct rte_flow_item *item;
3343         unsigned int has_vlan = 0;
3344
3345         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3346                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3347                         has_vlan = 1;
3348                         break;
3349                 }
3350         }
3351         if (has_vlan)
3352                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3353                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3354         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3355                                MLX5_EXPANSION_ROOT_OUTER;
3356 }
3357
3358 /**
3359  *  Get layer flags from the prefix flow.
3360  *
3361  *  Some flows may be split to several subflows, the prefix subflow gets the
3362  *  match items and the suffix sub flow gets the actions.
3363  *  Some actions need the user defined match item flags to get the detail for
3364  *  the action.
3365  *  This function helps the suffix flow to get the item layer flags from prefix
3366  *  subflow.
3367  *
3368  * @param[in] dev_flow
3369  *   Pointer the created preifx subflow.
3370  *
3371  * @return
3372  *   The layers get from prefix subflow.
3373  */
3374 static inline uint64_t
3375 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3376 {
3377         uint64_t layers = 0;
3378
3379         /*
3380          * Layers bits could be localization, but usually the compiler will
3381          * help to do the optimization work for source code.
3382          * If no decap actions, use the layers directly.
3383          */
3384         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3385                 return dev_flow->handle->layers;
3386         /* Convert L3 layers with decap action. */
3387         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3388                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3389         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3390                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3391         /* Convert L4 layers with decap action.  */
3392         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3393                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3394         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3395                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3396         return layers;
3397 }
3398
3399 /**
3400  * Get metadata split action information.
3401  *
3402  * @param[in] actions
3403  *   Pointer to the list of actions.
3404  * @param[out] qrss
3405  *   Pointer to the return pointer.
3406  * @param[out] qrss_type
3407  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3408  *   if no QUEUE/RSS is found.
3409  * @param[out] encap_idx
3410  *   Pointer to the index of the encap action if exists, otherwise the last
3411  *   action index.
3412  *
3413  * @return
3414  *   Total number of actions.
3415  */
3416 static int
3417 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3418                                        const struct rte_flow_action **qrss,
3419                                        int *encap_idx)
3420 {
3421         const struct rte_flow_action_raw_encap *raw_encap;
3422         int actions_n = 0;
3423         int raw_decap_idx = -1;
3424
3425         *encap_idx = -1;
3426         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3427                 switch (actions->type) {
3428                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3429                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3430                         *encap_idx = actions_n;
3431                         break;
3432                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3433                         raw_decap_idx = actions_n;
3434                         break;
3435                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3436                         raw_encap = actions->conf;
3437                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3438                                 *encap_idx = raw_decap_idx != -1 ?
3439                                                       raw_decap_idx : actions_n;
3440                         break;
3441                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3442                 case RTE_FLOW_ACTION_TYPE_RSS:
3443                         *qrss = actions;
3444                         break;
3445                 default:
3446                         break;
3447                 }
3448                 actions_n++;
3449         }
3450         if (*encap_idx == -1)
3451                 *encap_idx = actions_n;
3452         /* Count RTE_FLOW_ACTION_TYPE_END. */
3453         return actions_n + 1;
3454 }
3455
3456 /**
3457  * Check meter action from the action list.
3458  *
3459  * @param[in] actions
3460  *   Pointer to the list of actions.
3461  * @param[out] mtr
3462  *   Pointer to the meter exist flag.
3463  *
3464  * @return
3465  *   Total number of actions.
3466  */
3467 static int
3468 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3469 {
3470         int actions_n = 0;
3471
3472         MLX5_ASSERT(mtr);
3473         *mtr = 0;
3474         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3475                 switch (actions->type) {
3476                 case RTE_FLOW_ACTION_TYPE_METER:
3477                         *mtr = 1;
3478                         break;
3479                 default:
3480                         break;
3481                 }
3482                 actions_n++;
3483         }
3484         /* Count RTE_FLOW_ACTION_TYPE_END. */
3485         return actions_n + 1;
3486 }
3487
3488 /**
3489  * Check if the flow should be split due to hairpin.
3490  * The reason for the split is that in current HW we can't
3491  * support encap and push-vlan on Rx, so if a flow contains
3492  * these actions we move it to Tx.
3493  *
3494  * @param dev
3495  *   Pointer to Ethernet device.
3496  * @param[in] attr
3497  *   Flow rule attributes.
3498  * @param[in] actions
3499  *   Associated actions (list terminated by the END action).
3500  *
3501  * @return
3502  *   > 0 the number of actions and the flow should be split,
3503  *   0 when no split required.
3504  */
3505 static int
3506 flow_check_hairpin_split(struct rte_eth_dev *dev,
3507                          const struct rte_flow_attr *attr,
3508                          const struct rte_flow_action actions[])
3509 {
3510         int queue_action = 0;
3511         int action_n = 0;
3512         int split = 0;
3513         const struct rte_flow_action_queue *queue;
3514         const struct rte_flow_action_rss *rss;
3515         const struct rte_flow_action_raw_encap *raw_encap;
3516         const struct rte_eth_hairpin_conf *conf;
3517
3518         if (!attr->ingress)
3519                 return 0;
3520         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3521                 switch (actions->type) {
3522                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3523                         queue = actions->conf;
3524                         if (queue == NULL)
3525                                 return 0;
3526                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3527                         if (conf != NULL && !!conf->tx_explicit)
3528                                 return 0;
3529                         queue_action = 1;
3530                         action_n++;
3531                         break;
3532                 case RTE_FLOW_ACTION_TYPE_RSS:
3533                         rss = actions->conf;
3534                         if (rss == NULL || rss->queue_num == 0)
3535                                 return 0;
3536                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3537                         if (conf != NULL && !!conf->tx_explicit)
3538                                 return 0;
3539                         queue_action = 1;
3540                         action_n++;
3541                         break;
3542                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3543                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3544                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3545                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3546                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3547                         split++;
3548                         action_n++;
3549                         break;
3550                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3551                         raw_encap = actions->conf;
3552                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3553                                 split++;
3554                         action_n++;
3555                         break;
3556                 default:
3557                         action_n++;
3558                         break;
3559                 }
3560         }
3561         if (split && queue_action)
3562                 return action_n;
3563         return 0;
3564 }
3565
3566 /* Declare flow create/destroy prototype in advance. */
3567 static uint32_t
3568 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3569                  const struct rte_flow_attr *attr,
3570                  const struct rte_flow_item items[],
3571                  const struct rte_flow_action actions[],
3572                  bool external, struct rte_flow_error *error);
3573
3574 static void
3575 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3576                   uint32_t flow_idx);
3577
3578 int
3579 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
3580                       struct mlx5_hlist_entry *entry,
3581                       uint64_t key, void *cb_ctx __rte_unused)
3582 {
3583         struct mlx5_flow_mreg_copy_resource *mcp_res =
3584                 container_of(entry, typeof(*mcp_res), hlist_ent);
3585
3586         return mcp_res->mark_id != key;
3587 }
3588
3589 struct mlx5_hlist_entry *
3590 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3591                        void *cb_ctx)
3592 {
3593         struct rte_eth_dev *dev = list->ctx;
3594         struct mlx5_priv *priv = dev->data->dev_private;
3595         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3596         struct mlx5_flow_mreg_copy_resource *mcp_res;
3597         struct rte_flow_error *error = ctx->error;
3598         uint32_t idx = 0;
3599         int ret;
3600         uint32_t mark_id = key;
3601         struct rte_flow_attr attr = {
3602                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3603                 .ingress = 1,
3604         };
3605         struct mlx5_rte_flow_item_tag tag_spec = {
3606                 .data = mark_id,
3607         };
3608         struct rte_flow_item items[] = {
3609                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3610         };
3611         struct rte_flow_action_mark ftag = {
3612                 .id = mark_id,
3613         };
3614         struct mlx5_flow_action_copy_mreg cp_mreg = {
3615                 .dst = REG_B,
3616                 .src = REG_NON,
3617         };
3618         struct rte_flow_action_jump jump = {
3619                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3620         };
3621         struct rte_flow_action actions[] = {
3622                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3623         };
3624
3625         /* Fill the register fileds in the flow. */
3626         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3627         if (ret < 0)
3628                 return NULL;
3629         tag_spec.id = ret;
3630         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3631         if (ret < 0)
3632                 return NULL;
3633         cp_mreg.src = ret;
3634         /* Provide the full width of FLAG specific value. */
3635         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3636                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3637         /* Build a new flow. */
3638         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3639                 items[0] = (struct rte_flow_item){
3640                         .type = (enum rte_flow_item_type)
3641                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3642                         .spec = &tag_spec,
3643                 };
3644                 items[1] = (struct rte_flow_item){
3645                         .type = RTE_FLOW_ITEM_TYPE_END,
3646                 };
3647                 actions[0] = (struct rte_flow_action){
3648                         .type = (enum rte_flow_action_type)
3649                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3650                         .conf = &ftag,
3651                 };
3652                 actions[1] = (struct rte_flow_action){
3653                         .type = (enum rte_flow_action_type)
3654                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3655                         .conf = &cp_mreg,
3656                 };
3657                 actions[2] = (struct rte_flow_action){
3658                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3659                         .conf = &jump,
3660                 };
3661                 actions[3] = (struct rte_flow_action){
3662                         .type = RTE_FLOW_ACTION_TYPE_END,
3663                 };
3664         } else {
3665                 /* Default rule, wildcard match. */
3666                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3667                 items[0] = (struct rte_flow_item){
3668                         .type = RTE_FLOW_ITEM_TYPE_END,
3669                 };
3670                 actions[0] = (struct rte_flow_action){
3671                         .type = (enum rte_flow_action_type)
3672                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3673                         .conf = &cp_mreg,
3674                 };
3675                 actions[1] = (struct rte_flow_action){
3676                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3677                         .conf = &jump,
3678                 };
3679                 actions[2] = (struct rte_flow_action){
3680                         .type = RTE_FLOW_ACTION_TYPE_END,
3681                 };
3682         }
3683         /* Build a new entry. */
3684         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3685         if (!mcp_res) {
3686                 rte_errno = ENOMEM;
3687                 return NULL;
3688         }
3689         mcp_res->idx = idx;
3690         mcp_res->mark_id = mark_id;
3691         /*
3692          * The copy Flows are not included in any list. There
3693          * ones are referenced from other Flows and can not
3694          * be applied, removed, deleted in ardbitrary order
3695          * by list traversing.
3696          */
3697         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3698                                          actions, false, error);
3699         if (!mcp_res->rix_flow) {
3700                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
3701                 return NULL;
3702         }
3703         return &mcp_res->hlist_ent;
3704 }
3705
3706 /**
3707  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3708  *
3709  * As mark_id is unique, if there's already a registered flow for the mark_id,
3710  * return by increasing the reference counter of the resource. Otherwise, create
3711  * the resource (mcp_res) and flow.
3712  *
3713  * Flow looks like,
3714  *   - If ingress port is ANY and reg_c[1] is mark_id,
3715  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3716  *
3717  * For default flow (zero mark_id), flow is like,
3718  *   - If ingress port is ANY,
3719  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3720  *
3721  * @param dev
3722  *   Pointer to Ethernet device.
3723  * @param mark_id
3724  *   ID of MARK action, zero means default flow for META.
3725  * @param[out] error
3726  *   Perform verbose error reporting if not NULL.
3727  *
3728  * @return
3729  *   Associated resource on success, NULL otherwise and rte_errno is set.
3730  */
3731 static struct mlx5_flow_mreg_copy_resource *
3732 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3733                           struct rte_flow_error *error)
3734 {
3735         struct mlx5_priv *priv = dev->data->dev_private;
3736         struct mlx5_hlist_entry *entry;
3737         struct mlx5_flow_cb_ctx ctx = {
3738                 .dev = dev,
3739                 .error = error,
3740         };
3741
3742         /* Check if already registered. */
3743         MLX5_ASSERT(priv->mreg_cp_tbl);
3744         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
3745         if (!entry)
3746                 return NULL;
3747         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
3748                             hlist_ent);
3749 }
3750
3751 void
3752 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
3753 {
3754         struct mlx5_flow_mreg_copy_resource *mcp_res =
3755                 container_of(entry, typeof(*mcp_res), hlist_ent);
3756         struct rte_eth_dev *dev = list->ctx;
3757         struct mlx5_priv *priv = dev->data->dev_private;
3758
3759         MLX5_ASSERT(mcp_res->rix_flow);
3760         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3761         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3762 }
3763
3764 /**
3765  * Release flow in RX_CP_TBL.
3766  *
3767  * @param dev
3768  *   Pointer to Ethernet device.
3769  * @flow
3770  *   Parent flow for wich copying is provided.
3771  */
3772 static void
3773 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3774                           struct rte_flow *flow)
3775 {
3776         struct mlx5_flow_mreg_copy_resource *mcp_res;
3777         struct mlx5_priv *priv = dev->data->dev_private;
3778
3779         if (!flow->rix_mreg_copy)
3780                 return;
3781         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3782                                  flow->rix_mreg_copy);
3783         if (!mcp_res || !priv->mreg_cp_tbl)
3784                 return;
3785         MLX5_ASSERT(mcp_res->rix_flow);
3786         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3787         flow->rix_mreg_copy = 0;
3788 }
3789
3790 /**
3791  * Remove the default copy action from RX_CP_TBL.
3792  *
3793  * This functions is called in the mlx5_dev_start(). No thread safe
3794  * is guaranteed.
3795  *
3796  * @param dev
3797  *   Pointer to Ethernet device.
3798  */
3799 static void
3800 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3801 {
3802         struct mlx5_hlist_entry *entry;
3803         struct mlx5_priv *priv = dev->data->dev_private;
3804
3805         /* Check if default flow is registered. */
3806         if (!priv->mreg_cp_tbl)
3807                 return;
3808         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
3809                                   MLX5_DEFAULT_COPY_ID, NULL);
3810         if (!entry)
3811                 return;
3812         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
3813 }
3814
3815 /**
3816  * Add the default copy action in in RX_CP_TBL.
3817  *
3818  * This functions is called in the mlx5_dev_start(). No thread safe
3819  * is guaranteed.
3820  *
3821  * @param dev
3822  *   Pointer to Ethernet device.
3823  * @param[out] error
3824  *   Perform verbose error reporting if not NULL.
3825  *
3826  * @return
3827  *   0 for success, negative value otherwise and rte_errno is set.
3828  */
3829 static int
3830 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3831                                   struct rte_flow_error *error)
3832 {
3833         struct mlx5_priv *priv = dev->data->dev_private;
3834         struct mlx5_flow_mreg_copy_resource *mcp_res;
3835
3836         /* Check whether extensive metadata feature is engaged. */
3837         if (!priv->config.dv_flow_en ||
3838             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3839             !mlx5_flow_ext_mreg_supported(dev) ||
3840             !priv->sh->dv_regc0_mask)
3841                 return 0;
3842         /*
3843          * Add default mreg copy flow may be called multiple time, but
3844          * only be called once in stop. Avoid register it twice.
3845          */
3846         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
3847                 return 0;
3848         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3849         if (!mcp_res)
3850                 return -rte_errno;
3851         return 0;
3852 }
3853
3854 /**
3855  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3856  *
3857  * All the flow having Q/RSS action should be split by
3858  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3859  * performs the following,
3860  *   - CQE->flow_tag := reg_c[1] (MARK)
3861  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3862  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3863  * but there should be a flow per each MARK ID set by MARK action.
3864  *
3865  * For the aforementioned reason, if there's a MARK action in flow's action
3866  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3867  * the MARK ID to CQE's flow_tag like,
3868  *   - If reg_c[1] is mark_id,
3869  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3870  *
3871  * For SET_META action which stores value in reg_c[0], as the destination is
3872  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3873  * MARK ID means the default flow. The default flow looks like,
3874  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3875  *
3876  * @param dev
3877  *   Pointer to Ethernet device.
3878  * @param flow
3879  *   Pointer to flow structure.
3880  * @param[in] actions
3881  *   Pointer to the list of actions.
3882  * @param[out] error
3883  *   Perform verbose error reporting if not NULL.
3884  *
3885  * @return
3886  *   0 on success, negative value otherwise and rte_errno is set.
3887  */
3888 static int
3889 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3890                             struct rte_flow *flow,
3891                             const struct rte_flow_action *actions,
3892                             struct rte_flow_error *error)
3893 {
3894         struct mlx5_priv *priv = dev->data->dev_private;
3895         struct mlx5_dev_config *config = &priv->config;
3896         struct mlx5_flow_mreg_copy_resource *mcp_res;
3897         const struct rte_flow_action_mark *mark;
3898
3899         /* Check whether extensive metadata feature is engaged. */
3900         if (!config->dv_flow_en ||
3901             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3902             !mlx5_flow_ext_mreg_supported(dev) ||
3903             !priv->sh->dv_regc0_mask)
3904                 return 0;
3905         /* Find MARK action. */
3906         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3907                 switch (actions->type) {
3908                 case RTE_FLOW_ACTION_TYPE_FLAG:
3909                         mcp_res = flow_mreg_add_copy_action
3910                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3911                         if (!mcp_res)
3912                                 return -rte_errno;
3913                         flow->rix_mreg_copy = mcp_res->idx;
3914                         return 0;
3915                 case RTE_FLOW_ACTION_TYPE_MARK:
3916                         mark = (const struct rte_flow_action_mark *)
3917                                 actions->conf;
3918                         mcp_res =
3919                                 flow_mreg_add_copy_action(dev, mark->id, error);
3920                         if (!mcp_res)
3921                                 return -rte_errno;
3922                         flow->rix_mreg_copy = mcp_res->idx;
3923                         return 0;
3924                 default:
3925                         break;
3926                 }
3927         }
3928         return 0;
3929 }
3930
3931 #define MLX5_MAX_SPLIT_ACTIONS 24
3932 #define MLX5_MAX_SPLIT_ITEMS 24
3933
3934 /**
3935  * Split the hairpin flow.
3936  * Since HW can't support encap and push-vlan on Rx, we move these
3937  * actions to Tx.
3938  * If the count action is after the encap then we also
3939  * move the count action. in this case the count will also measure
3940  * the outer bytes.
3941  *
3942  * @param dev
3943  *   Pointer to Ethernet device.
3944  * @param[in] actions
3945  *   Associated actions (list terminated by the END action).
3946  * @param[out] actions_rx
3947  *   Rx flow actions.
3948  * @param[out] actions_tx
3949  *   Tx flow actions..
3950  * @param[out] pattern_tx
3951  *   The pattern items for the Tx flow.
3952  * @param[out] flow_id
3953  *   The flow ID connected to this flow.
3954  *
3955  * @return
3956  *   0 on success.
3957  */
3958 static int
3959 flow_hairpin_split(struct rte_eth_dev *dev,
3960                    const struct rte_flow_action actions[],
3961                    struct rte_flow_action actions_rx[],
3962                    struct rte_flow_action actions_tx[],
3963                    struct rte_flow_item pattern_tx[],
3964                    uint32_t flow_id)
3965 {
3966         const struct rte_flow_action_raw_encap *raw_encap;
3967         const struct rte_flow_action_raw_decap *raw_decap;
3968         struct mlx5_rte_flow_action_set_tag *set_tag;
3969         struct rte_flow_action *tag_action;
3970         struct mlx5_rte_flow_item_tag *tag_item;
3971         struct rte_flow_item *item;
3972         char *addr;
3973         int encap = 0;
3974
3975         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3976                 switch (actions->type) {
3977                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3978                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3979                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3980                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3981                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3982                         rte_memcpy(actions_tx, actions,
3983                                sizeof(struct rte_flow_action));
3984                         actions_tx++;
3985                         break;
3986                 case RTE_FLOW_ACTION_TYPE_COUNT:
3987                         if (encap) {
3988                                 rte_memcpy(actions_tx, actions,
3989                                            sizeof(struct rte_flow_action));
3990                                 actions_tx++;
3991                         } else {
3992                                 rte_memcpy(actions_rx, actions,
3993                                            sizeof(struct rte_flow_action));
3994                                 actions_rx++;
3995                         }
3996                         break;
3997                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3998                         raw_encap = actions->conf;
3999                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4000                                 memcpy(actions_tx, actions,
4001                                        sizeof(struct rte_flow_action));
4002                                 actions_tx++;
4003                                 encap = 1;
4004                         } else {
4005                                 rte_memcpy(actions_rx, actions,
4006                                            sizeof(struct rte_flow_action));
4007                                 actions_rx++;
4008                         }
4009                         break;
4010                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4011                         raw_decap = actions->conf;
4012                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4013                                 memcpy(actions_tx, actions,
4014                                        sizeof(struct rte_flow_action));
4015                                 actions_tx++;
4016                         } else {
4017                                 rte_memcpy(actions_rx, actions,
4018                                            sizeof(struct rte_flow_action));
4019                                 actions_rx++;
4020                         }
4021                         break;
4022                 default:
4023                         rte_memcpy(actions_rx, actions,
4024                                    sizeof(struct rte_flow_action));
4025                         actions_rx++;
4026                         break;
4027                 }
4028         }
4029         /* Add set meta action and end action for the Rx flow. */
4030         tag_action = actions_rx;
4031         tag_action->type = (enum rte_flow_action_type)
4032                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4033         actions_rx++;
4034         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4035         actions_rx++;
4036         set_tag = (void *)actions_rx;
4037         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4038         MLX5_ASSERT(set_tag->id > REG_NON);
4039         set_tag->data = flow_id;
4040         tag_action->conf = set_tag;
4041         /* Create Tx item list. */
4042         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4043         addr = (void *)&pattern_tx[2];
4044         item = pattern_tx;
4045         item->type = (enum rte_flow_item_type)
4046                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4047         tag_item = (void *)addr;
4048         tag_item->data = flow_id;
4049         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4050         MLX5_ASSERT(set_tag->id > REG_NON);
4051         item->spec = tag_item;
4052         addr += sizeof(struct mlx5_rte_flow_item_tag);
4053         tag_item = (void *)addr;
4054         tag_item->data = UINT32_MAX;
4055         tag_item->id = UINT16_MAX;
4056         item->mask = tag_item;
4057         item->last = NULL;
4058         item++;
4059         item->type = RTE_FLOW_ITEM_TYPE_END;
4060         return 0;
4061 }
4062
4063 /**
4064  * The last stage of splitting chain, just creates the subflow
4065  * without any modification.
4066  *
4067  * @param[in] dev
4068  *   Pointer to Ethernet device.
4069  * @param[in] flow
4070  *   Parent flow structure pointer.
4071  * @param[in, out] sub_flow
4072  *   Pointer to return the created subflow, may be NULL.
4073  * @param[in] attr
4074  *   Flow rule attributes.
4075  * @param[in] items
4076  *   Pattern specification (list terminated by the END pattern item).
4077  * @param[in] actions
4078  *   Associated actions (list terminated by the END action).
4079  * @param[in] flow_split_info
4080  *   Pointer to flow split info structure.
4081  * @param[out] error
4082  *   Perform verbose error reporting if not NULL.
4083  * @return
4084  *   0 on success, negative value otherwise
4085  */
4086 static int
4087 flow_create_split_inner(struct rte_eth_dev *dev,
4088                         struct rte_flow *flow,
4089                         struct mlx5_flow **sub_flow,
4090                         const struct rte_flow_attr *attr,
4091                         const struct rte_flow_item items[],
4092                         const struct rte_flow_action actions[],
4093                         struct mlx5_flow_split_info *flow_split_info,
4094                         struct rte_flow_error *error)
4095 {
4096         struct mlx5_flow *dev_flow;
4097
4098         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4099                                     flow_split_info->flow_idx, error);
4100         if (!dev_flow)
4101                 return -rte_errno;
4102         dev_flow->flow = flow;
4103         dev_flow->external = flow_split_info->external;
4104         dev_flow->skip_scale = flow_split_info->skip_scale;
4105         /* Subflow object was created, we must include one in the list. */
4106         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4107                       dev_flow->handle, next);
4108         /*
4109          * If dev_flow is as one of the suffix flow, some actions in suffix
4110          * flow may need some user defined item layer flags, and pass the
4111          * Metadate rxq mark flag to suffix flow as well.
4112          */
4113         if (flow_split_info->prefix_layers)
4114                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4115         if (flow_split_info->prefix_mark)
4116                 dev_flow->handle->mark = 1;
4117         if (sub_flow)
4118                 *sub_flow = dev_flow;
4119         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4120 }
4121
4122 /**
4123  * Split the meter flow.
4124  *
4125  * As meter flow will split to three sub flow, other than meter
4126  * action, the other actions make sense to only meter accepts
4127  * the packet. If it need to be dropped, no other additional
4128  * actions should be take.
4129  *
4130  * One kind of special action which decapsulates the L3 tunnel
4131  * header will be in the prefix sub flow, as not to take the
4132  * L3 tunnel header into account.
4133  *
4134  * @param dev
4135  *   Pointer to Ethernet device.
4136  * @param[in] items
4137  *   Pattern specification (list terminated by the END pattern item).
4138  * @param[out] sfx_items
4139  *   Suffix flow match items (list terminated by the END pattern item).
4140  * @param[in] actions
4141  *   Associated actions (list terminated by the END action).
4142  * @param[out] actions_sfx
4143  *   Suffix flow actions.
4144  * @param[out] actions_pre
4145  *   Prefix flow actions.
4146  * @param[out] pattern_sfx
4147  *   The pattern items for the suffix flow.
4148  * @param[out] tag_sfx
4149  *   Pointer to suffix flow tag.
4150  *
4151  * @return
4152  *   0 on success.
4153  */
4154 static int
4155 flow_meter_split_prep(struct rte_eth_dev *dev,
4156                  const struct rte_flow_item items[],
4157                  struct rte_flow_item sfx_items[],
4158                  const struct rte_flow_action actions[],
4159                  struct rte_flow_action actions_sfx[],
4160                  struct rte_flow_action actions_pre[])
4161 {
4162         struct mlx5_priv *priv = dev->data->dev_private;
4163         struct rte_flow_action *tag_action = NULL;
4164         struct rte_flow_item *tag_item;
4165         struct mlx5_rte_flow_action_set_tag *set_tag;
4166         struct rte_flow_error error;
4167         const struct rte_flow_action_raw_encap *raw_encap;
4168         const struct rte_flow_action_raw_decap *raw_decap;
4169         struct mlx5_rte_flow_item_tag *tag_spec;
4170         struct mlx5_rte_flow_item_tag *tag_mask;
4171         uint32_t tag_id = 0;
4172         bool copy_vlan = false;
4173
4174         /* Prepare the actions for prefix and suffix flow. */
4175         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4176                 struct rte_flow_action **action_cur = NULL;
4177
4178                 switch (actions->type) {
4179                 case RTE_FLOW_ACTION_TYPE_METER:
4180                         /* Add the extra tag action first. */
4181                         tag_action = actions_pre;
4182                         tag_action->type = (enum rte_flow_action_type)
4183                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4184                         actions_pre++;
4185                         action_cur = &actions_pre;
4186                         break;
4187                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4188                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4189                         action_cur = &actions_pre;
4190                         break;
4191                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4192                         raw_encap = actions->conf;
4193                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4194                                 action_cur = &actions_pre;
4195                         break;
4196                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4197                         raw_decap = actions->conf;
4198                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4199                                 action_cur = &actions_pre;
4200                         break;
4201                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4202                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4203                         copy_vlan = true;
4204                         break;
4205                 default:
4206                         break;
4207                 }
4208                 if (!action_cur)
4209                         action_cur = &actions_sfx;
4210                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4211                 (*action_cur)++;
4212         }
4213         /* Add end action to the actions. */
4214         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4215         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4216         actions_pre++;
4217         /* Set the tag. */
4218         set_tag = (void *)actions_pre;
4219         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4220         mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4221                           &tag_id);
4222         if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
4223                 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
4224                         dev->data->port_id);
4225                 mlx5_ipool_free(priv->sh->ipool
4226                                 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
4227                 return 0;
4228         } else if (!tag_id) {
4229                 return 0;
4230         }
4231         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4232         assert(tag_action);
4233         tag_action->conf = set_tag;
4234         /* Prepare the suffix subflow items. */
4235         tag_item = sfx_items++;
4236         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4237                 int item_type = items->type;
4238
4239                 switch (item_type) {
4240                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4241                         memcpy(sfx_items, items, sizeof(*sfx_items));
4242                         sfx_items++;
4243                         break;
4244                 case RTE_FLOW_ITEM_TYPE_VLAN:
4245                         if (copy_vlan) {
4246                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4247                                 /*
4248                                  * Convert to internal match item, it is used
4249                                  * for vlan push and set vid.
4250                                  */
4251                                 sfx_items->type = (enum rte_flow_item_type)
4252                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4253                                 sfx_items++;
4254                         }
4255                         break;
4256                 default:
4257                         break;
4258                 }
4259         }
4260         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4261         sfx_items++;
4262         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4263         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4264         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4265         tag_mask = tag_spec + 1;
4266         tag_mask->data = 0xffffff00;
4267         tag_item->type = (enum rte_flow_item_type)
4268                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4269         tag_item->spec = tag_spec;
4270         tag_item->last = NULL;
4271         tag_item->mask = tag_mask;
4272         return tag_id;
4273 }
4274
4275 /**
4276  * Split action list having QUEUE/RSS for metadata register copy.
4277  *
4278  * Once Q/RSS action is detected in user's action list, the flow action
4279  * should be split in order to copy metadata registers, which will happen in
4280  * RX_CP_TBL like,
4281  *   - CQE->flow_tag := reg_c[1] (MARK)
4282  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4283  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4284  * This is because the last action of each flow must be a terminal action
4285  * (QUEUE, RSS or DROP).
4286  *
4287  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4288  * stored and kept in the mlx5_flow structure per each sub_flow.
4289  *
4290  * The Q/RSS action is replaced with,
4291  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4292  * And the following JUMP action is added at the end,
4293  *   - JUMP, to RX_CP_TBL.
4294  *
4295  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4296  * flow_create_split_metadata() routine. The flow will look like,
4297  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4298  *
4299  * @param dev
4300  *   Pointer to Ethernet device.
4301  * @param[out] split_actions
4302  *   Pointer to store split actions to jump to CP_TBL.
4303  * @param[in] actions
4304  *   Pointer to the list of original flow actions.
4305  * @param[in] qrss
4306  *   Pointer to the Q/RSS action.
4307  * @param[in] actions_n
4308  *   Number of original actions.
4309  * @param[out] error
4310  *   Perform verbose error reporting if not NULL.
4311  *
4312  * @return
4313  *   non-zero unique flow_id on success, otherwise 0 and
4314  *   error/rte_error are set.
4315  */
4316 static uint32_t
4317 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4318                           struct rte_flow_action *split_actions,
4319                           const struct rte_flow_action *actions,
4320                           const struct rte_flow_action *qrss,
4321                           int actions_n, struct rte_flow_error *error)
4322 {
4323         struct mlx5_priv *priv = dev->data->dev_private;
4324         struct mlx5_rte_flow_action_set_tag *set_tag;
4325         struct rte_flow_action_jump *jump;
4326         const int qrss_idx = qrss - actions;
4327         uint32_t flow_id = 0;
4328         int ret = 0;
4329
4330         /*
4331          * Given actions will be split
4332          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4333          * - Add jump to mreg CP_TBL.
4334          * As a result, there will be one more action.
4335          */
4336         ++actions_n;
4337         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4338         set_tag = (void *)(split_actions + actions_n);
4339         /*
4340          * If tag action is not set to void(it means we are not the meter
4341          * suffix flow), add the tag action. Since meter suffix flow already
4342          * has the tag added.
4343          */
4344         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4345                 /*
4346                  * Allocate the new subflow ID. This one is unique within
4347                  * device and not shared with representors. Otherwise,
4348                  * we would have to resolve multi-thread access synch
4349                  * issue. Each flow on the shared device is appended
4350                  * with source vport identifier, so the resulting
4351                  * flows will be unique in the shared (by master and
4352                  * representors) domain even if they have coinciding
4353                  * IDs.
4354                  */
4355                 mlx5_ipool_malloc(priv->sh->ipool
4356                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4357                 if (!flow_id)
4358                         return rte_flow_error_set(error, ENOMEM,
4359                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4360                                                   NULL, "can't allocate id "
4361                                                   "for split Q/RSS subflow");
4362                 /* Internal SET_TAG action to set flow ID. */
4363                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4364                         .data = flow_id,
4365                 };
4366                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4367                 if (ret < 0)
4368                         return ret;
4369                 set_tag->id = ret;
4370                 /* Construct new actions array. */
4371                 /* Replace QUEUE/RSS action. */
4372                 split_actions[qrss_idx] = (struct rte_flow_action){
4373                         .type = (enum rte_flow_action_type)
4374                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4375                         .conf = set_tag,
4376                 };
4377         }
4378         /* JUMP action to jump to mreg copy table (CP_TBL). */
4379         jump = (void *)(set_tag + 1);
4380         *jump = (struct rte_flow_action_jump){
4381                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4382         };
4383         split_actions[actions_n - 2] = (struct rte_flow_action){
4384                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4385                 .conf = jump,
4386         };
4387         split_actions[actions_n - 1] = (struct rte_flow_action){
4388                 .type = RTE_FLOW_ACTION_TYPE_END,
4389         };
4390         return flow_id;
4391 }
4392
4393 /**
4394  * Extend the given action list for Tx metadata copy.
4395  *
4396  * Copy the given action list to the ext_actions and add flow metadata register
4397  * copy action in order to copy reg_a set by WQE to reg_c[0].
4398  *
4399  * @param[out] ext_actions
4400  *   Pointer to the extended action list.
4401  * @param[in] actions
4402  *   Pointer to the list of actions.
4403  * @param[in] actions_n
4404  *   Number of actions in the list.
4405  * @param[out] error
4406  *   Perform verbose error reporting if not NULL.
4407  * @param[in] encap_idx
4408  *   The encap action inndex.
4409  *
4410  * @return
4411  *   0 on success, negative value otherwise
4412  */
4413 static int
4414 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4415                        struct rte_flow_action *ext_actions,
4416                        const struct rte_flow_action *actions,
4417                        int actions_n, struct rte_flow_error *error,
4418                        int encap_idx)
4419 {
4420         struct mlx5_flow_action_copy_mreg *cp_mreg =
4421                 (struct mlx5_flow_action_copy_mreg *)
4422                         (ext_actions + actions_n + 1);
4423         int ret;
4424
4425         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4426         if (ret < 0)
4427                 return ret;
4428         cp_mreg->dst = ret;
4429         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4430         if (ret < 0)
4431                 return ret;
4432         cp_mreg->src = ret;
4433         if (encap_idx != 0)
4434                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4435         if (encap_idx == actions_n - 1) {
4436                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4437                         .type = (enum rte_flow_action_type)
4438                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4439                         .conf = cp_mreg,
4440                 };
4441                 ext_actions[actions_n] = (struct rte_flow_action){
4442                         .type = RTE_FLOW_ACTION_TYPE_END,
4443                 };
4444         } else {
4445                 ext_actions[encap_idx] = (struct rte_flow_action){
4446                         .type = (enum rte_flow_action_type)
4447                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4448                         .conf = cp_mreg,
4449                 };
4450                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4451                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4452         }
4453         return 0;
4454 }
4455
4456 /**
4457  * Check the match action from the action list.
4458  *
4459  * @param[in] actions
4460  *   Pointer to the list of actions.
4461  * @param[in] attr
4462  *   Flow rule attributes.
4463  * @param[in] action
4464  *   The action to be check if exist.
4465  * @param[out] match_action_pos
4466  *   Pointer to the position of the matched action if exists, otherwise is -1.
4467  * @param[out] qrss_action_pos
4468  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4469  *
4470  * @return
4471  *   > 0 the total number of actions.
4472  *   0 if not found match action in action list.
4473  */
4474 static int
4475 flow_check_match_action(const struct rte_flow_action actions[],
4476                         const struct rte_flow_attr *attr,
4477                         enum rte_flow_action_type action,
4478                         int *match_action_pos, int *qrss_action_pos)
4479 {
4480         const struct rte_flow_action_sample *sample;
4481         int actions_n = 0;
4482         int jump_flag = 0;
4483         uint32_t ratio = 0;
4484         int sub_type = 0;
4485         int flag = 0;
4486
4487         *match_action_pos = -1;
4488         *qrss_action_pos = -1;
4489         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4490                 if (actions->type == action) {
4491                         flag = 1;
4492                         *match_action_pos = actions_n;
4493                 }
4494                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4495                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4496                         *qrss_action_pos = actions_n;
4497                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4498                         jump_flag = 1;
4499                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4500                         sample = actions->conf;
4501                         ratio = sample->ratio;
4502                         sub_type = ((const struct rte_flow_action *)
4503                                         (sample->actions))->type;
4504                 }
4505                 actions_n++;
4506         }
4507         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4508                 if (ratio == 1) {
4509                         /* JUMP Action not support for Mirroring;
4510                          * Mirroring support multi-destination;
4511                          */
4512                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4513                                 flag = 0;
4514                 }
4515         }
4516         /* Count RTE_FLOW_ACTION_TYPE_END. */
4517         return flag ? actions_n + 1 : 0;
4518 }
4519
4520 #define SAMPLE_SUFFIX_ITEM 2
4521
4522 /**
4523  * Split the sample flow.
4524  *
4525  * As sample flow will split to two sub flow, sample flow with
4526  * sample action, the other actions will move to new suffix flow.
4527  *
4528  * Also add unique tag id with tag action in the sample flow,
4529  * the same tag id will be as match in the suffix flow.
4530  *
4531  * @param dev
4532  *   Pointer to Ethernet device.
4533  * @param[in] fdb_tx
4534  *   FDB egress flow flag.
4535  * @param[out] sfx_items
4536  *   Suffix flow match items (list terminated by the END pattern item).
4537  * @param[in] actions
4538  *   Associated actions (list terminated by the END action).
4539  * @param[out] actions_sfx
4540  *   Suffix flow actions.
4541  * @param[out] actions_pre
4542  *   Prefix flow actions.
4543  * @param[in] actions_n
4544  *  The total number of actions.
4545  * @param[in] sample_action_pos
4546  *   The sample action position.
4547  * @param[in] qrss_action_pos
4548  *   The Queue/RSS action position.
4549  * @param[out] error
4550  *   Perform verbose error reporting if not NULL.
4551  *
4552  * @return
4553  *   0 on success, or unique flow_id, a negative errno value
4554  *   otherwise and rte_errno is set.
4555  */
4556 static int
4557 flow_sample_split_prep(struct rte_eth_dev *dev,
4558                        uint32_t fdb_tx,
4559                        struct rte_flow_item sfx_items[],
4560                        const struct rte_flow_action actions[],
4561                        struct rte_flow_action actions_sfx[],
4562                        struct rte_flow_action actions_pre[],
4563                        int actions_n,
4564                        int sample_action_pos,
4565                        int qrss_action_pos,
4566                        struct rte_flow_error *error)
4567 {
4568         struct mlx5_priv *priv = dev->data->dev_private;
4569         struct mlx5_rte_flow_action_set_tag *set_tag;
4570         struct mlx5_rte_flow_item_tag *tag_spec;
4571         struct mlx5_rte_flow_item_tag *tag_mask;
4572         uint32_t tag_id = 0;
4573         int index;
4574         int ret;
4575
4576         if (sample_action_pos < 0)
4577                 return rte_flow_error_set(error, EINVAL,
4578                                           RTE_FLOW_ERROR_TYPE_ACTION,
4579                                           NULL, "invalid position of sample "
4580                                           "action in list");
4581         if (!fdb_tx) {
4582                 /* Prepare the prefix tag action. */
4583                 set_tag = (void *)(actions_pre + actions_n + 1);
4584                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4585                 if (ret < 0)
4586                         return ret;
4587                 set_tag->id = ret;
4588                 mlx5_ipool_malloc(priv->sh->ipool
4589                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
4590                 set_tag->data = tag_id;
4591                 /* Prepare the suffix subflow items. */
4592                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4593                 tag_spec->data = tag_id;
4594                 tag_spec->id = set_tag->id;
4595                 tag_mask = tag_spec + 1;
4596                 tag_mask->data = UINT32_MAX;
4597                 sfx_items[0] = (struct rte_flow_item){
4598                         .type = (enum rte_flow_item_type)
4599                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4600                         .spec = tag_spec,
4601                         .last = NULL,
4602                         .mask = tag_mask,
4603                 };
4604                 sfx_items[1] = (struct rte_flow_item){
4605                         .type = (enum rte_flow_item_type)
4606                                 RTE_FLOW_ITEM_TYPE_END,
4607                 };
4608         }
4609         /* Prepare the actions for prefix and suffix flow. */
4610         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4611                 index = qrss_action_pos;
4612                 /* Put the preceding the Queue/RSS action into prefix flow. */
4613                 if (index != 0)
4614                         memcpy(actions_pre, actions,
4615                                sizeof(struct rte_flow_action) * index);
4616                 /* Put others preceding the sample action into prefix flow. */
4617                 if (sample_action_pos > index + 1)
4618                         memcpy(actions_pre + index, actions + index + 1,
4619                                sizeof(struct rte_flow_action) *
4620                                (sample_action_pos - index - 1));
4621                 index = sample_action_pos - 1;
4622                 /* Put Queue/RSS action into Suffix flow. */
4623                 memcpy(actions_sfx, actions + qrss_action_pos,
4624                        sizeof(struct rte_flow_action));
4625                 actions_sfx++;
4626         } else {
4627                 index = sample_action_pos;
4628                 if (index != 0)
4629                         memcpy(actions_pre, actions,
4630                                sizeof(struct rte_flow_action) * index);
4631         }
4632         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4633         if (!fdb_tx) {
4634                 actions_pre[index++] =
4635                         (struct rte_flow_action){
4636                         .type = (enum rte_flow_action_type)
4637                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4638                         .conf = set_tag,
4639                 };
4640         }
4641         memcpy(actions_pre + index, actions + sample_action_pos,
4642                sizeof(struct rte_flow_action));
4643         index += 1;
4644         actions_pre[index] = (struct rte_flow_action){
4645                 .type = (enum rte_flow_action_type)
4646                         RTE_FLOW_ACTION_TYPE_END,
4647         };
4648         /* Put the actions after sample into Suffix flow. */
4649         memcpy(actions_sfx, actions + sample_action_pos + 1,
4650                sizeof(struct rte_flow_action) *
4651                (actions_n - sample_action_pos - 1));
4652         return tag_id;
4653 }
4654
4655 /**
4656  * The splitting for metadata feature.
4657  *
4658  * - Q/RSS action on NIC Rx should be split in order to pass by
4659  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4660  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4661  *
4662  * - All the actions on NIC Tx should have a mreg copy action to
4663  *   copy reg_a from WQE to reg_c[0].
4664  *
4665  * @param dev
4666  *   Pointer to Ethernet device.
4667  * @param[in] flow
4668  *   Parent flow structure pointer.
4669  * @param[in] attr
4670  *   Flow rule attributes.
4671  * @param[in] items
4672  *   Pattern specification (list terminated by the END pattern item).
4673  * @param[in] actions
4674  *   Associated actions (list terminated by the END action).
4675  * @param[in] flow_split_info
4676  *   Pointer to flow split info structure.
4677  * @param[out] error
4678  *   Perform verbose error reporting if not NULL.
4679  * @return
4680  *   0 on success, negative value otherwise
4681  */
4682 static int
4683 flow_create_split_metadata(struct rte_eth_dev *dev,
4684                            struct rte_flow *flow,
4685                            const struct rte_flow_attr *attr,
4686                            const struct rte_flow_item items[],
4687                            const struct rte_flow_action actions[],
4688                            struct mlx5_flow_split_info *flow_split_info,
4689                            struct rte_flow_error *error)
4690 {
4691         struct mlx5_priv *priv = dev->data->dev_private;
4692         struct mlx5_dev_config *config = &priv->config;
4693         const struct rte_flow_action *qrss = NULL;
4694         struct rte_flow_action *ext_actions = NULL;
4695         struct mlx5_flow *dev_flow = NULL;
4696         uint32_t qrss_id = 0;
4697         int mtr_sfx = 0;
4698         size_t act_size;
4699         int actions_n;
4700         int encap_idx;
4701         int ret;
4702
4703         /* Check whether extensive metadata feature is engaged. */
4704         if (!config->dv_flow_en ||
4705             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4706             !mlx5_flow_ext_mreg_supported(dev))
4707                 return flow_create_split_inner(dev, flow, NULL, attr, items,
4708                                                actions, flow_split_info, error);
4709         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4710                                                            &encap_idx);
4711         if (qrss) {
4712                 /* Exclude hairpin flows from splitting. */
4713                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4714                         const struct rte_flow_action_queue *queue;
4715
4716                         queue = qrss->conf;
4717                         if (mlx5_rxq_get_type(dev, queue->index) ==
4718                             MLX5_RXQ_TYPE_HAIRPIN)
4719                                 qrss = NULL;
4720                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4721                         const struct rte_flow_action_rss *rss;
4722
4723                         rss = qrss->conf;
4724                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4725                             MLX5_RXQ_TYPE_HAIRPIN)
4726                                 qrss = NULL;
4727                 }
4728         }
4729         if (qrss) {
4730                 /* Check if it is in meter suffix table. */
4731                 mtr_sfx = attr->group == (attr->transfer ?
4732                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4733                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4734                 /*
4735                  * Q/RSS action on NIC Rx should be split in order to pass by
4736                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4737                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4738                  */
4739                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4740                            sizeof(struct rte_flow_action_set_tag) +
4741                            sizeof(struct rte_flow_action_jump);
4742                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4743                                           SOCKET_ID_ANY);
4744                 if (!ext_actions)
4745                         return rte_flow_error_set(error, ENOMEM,
4746                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4747                                                   NULL, "no memory to split "
4748                                                   "metadata flow");
4749                 /*
4750                  * If we are the suffix flow of meter, tag already exist.
4751                  * Set the tag action to void.
4752                  */
4753                 if (mtr_sfx)
4754                         ext_actions[qrss - actions].type =
4755                                                 RTE_FLOW_ACTION_TYPE_VOID;
4756                 else
4757                         ext_actions[qrss - actions].type =
4758                                                 (enum rte_flow_action_type)
4759                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4760                 /*
4761                  * Create the new actions list with removed Q/RSS action
4762                  * and appended set tag and jump to register copy table
4763                  * (RX_CP_TBL). We should preallocate unique tag ID here
4764                  * in advance, because it is needed for set tag action.
4765                  */
4766                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4767                                                     qrss, actions_n, error);
4768                 if (!mtr_sfx && !qrss_id) {
4769                         ret = -rte_errno;
4770                         goto exit;
4771                 }
4772         } else if (attr->egress && !attr->transfer) {
4773                 /*
4774                  * All the actions on NIC Tx should have a metadata register
4775                  * copy action to copy reg_a from WQE to reg_c[meta]
4776                  */
4777                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4778                            sizeof(struct mlx5_flow_action_copy_mreg);
4779                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4780                                           SOCKET_ID_ANY);
4781                 if (!ext_actions)
4782                         return rte_flow_error_set(error, ENOMEM,
4783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4784                                                   NULL, "no memory to split "
4785                                                   "metadata flow");
4786                 /* Create the action list appended with copy register. */
4787                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4788                                              actions_n, error, encap_idx);
4789                 if (ret < 0)
4790                         goto exit;
4791         }
4792         /* Add the unmodified original or prefix subflow. */
4793         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
4794                                       items, ext_actions ? ext_actions :
4795                                       actions, flow_split_info, error);
4796         if (ret < 0)
4797                 goto exit;
4798         MLX5_ASSERT(dev_flow);
4799         if (qrss) {
4800                 const struct rte_flow_attr q_attr = {
4801                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4802                         .ingress = 1,
4803                 };
4804                 /* Internal PMD action to set register. */
4805                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4806                         .data = qrss_id,
4807                         .id = REG_NON,
4808                 };
4809                 struct rte_flow_item q_items[] = {
4810                         {
4811                                 .type = (enum rte_flow_item_type)
4812                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4813                                 .spec = &q_tag_spec,
4814                                 .last = NULL,
4815                                 .mask = NULL,
4816                         },
4817                         {
4818                                 .type = RTE_FLOW_ITEM_TYPE_END,
4819                         },
4820                 };
4821                 struct rte_flow_action q_actions[] = {
4822                         {
4823                                 .type = qrss->type,
4824                                 .conf = qrss->conf,
4825                         },
4826                         {
4827                                 .type = RTE_FLOW_ACTION_TYPE_END,
4828                         },
4829                 };
4830                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4831
4832                 /*
4833                  * Configure the tag item only if there is no meter subflow.
4834                  * Since tag is already marked in the meter suffix subflow
4835                  * we can just use the meter suffix items as is.
4836                  */
4837                 if (qrss_id) {
4838                         /* Not meter subflow. */
4839                         MLX5_ASSERT(!mtr_sfx);
4840                         /*
4841                          * Put unique id in prefix flow due to it is destroyed
4842                          * after suffix flow and id will be freed after there
4843                          * is no actual flows with this id and identifier
4844                          * reallocation becomes possible (for example, for
4845                          * other flows in other threads).
4846                          */
4847                         dev_flow->handle->split_flow_id = qrss_id;
4848                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4849                                                    error);
4850                         if (ret < 0)
4851                                 goto exit;
4852                         q_tag_spec.id = ret;
4853                 }
4854                 dev_flow = NULL;
4855                 /* Add suffix subflow to execute Q/RSS. */
4856                 flow_split_info->prefix_layers = layers;
4857                 flow_split_info->prefix_mark = 0;
4858                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4859                                               &q_attr, mtr_sfx ? items :
4860                                               q_items, q_actions,
4861                                               flow_split_info, error);
4862                 if (ret < 0)
4863                         goto exit;
4864                 /* qrss ID should be freed if failed. */
4865                 qrss_id = 0;
4866                 MLX5_ASSERT(dev_flow);
4867         }
4868
4869 exit:
4870         /*
4871          * We do not destroy the partially created sub_flows in case of error.
4872          * These ones are included into parent flow list and will be destroyed
4873          * by flow_drv_destroy.
4874          */
4875         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4876                         qrss_id);
4877         mlx5_free(ext_actions);
4878         return ret;
4879 }
4880
4881 /**
4882  * The splitting for meter feature.
4883  *
4884  * - The meter flow will be split to two flows as prefix and
4885  *   suffix flow. The packets make sense only it pass the prefix
4886  *   meter action.
4887  *
4888  * - Reg_C_5 is used for the packet to match betweend prefix and
4889  *   suffix flow.
4890  *
4891  * @param dev
4892  *   Pointer to Ethernet device.
4893  * @param[in] flow
4894  *   Parent flow structure pointer.
4895  * @param[in] attr
4896  *   Flow rule attributes.
4897  * @param[in] items
4898  *   Pattern specification (list terminated by the END pattern item).
4899  * @param[in] actions
4900  *   Associated actions (list terminated by the END action).
4901  * @param[in] flow_split_info
4902  *   Pointer to flow split info structure.
4903  * @param[out] error
4904  *   Perform verbose error reporting if not NULL.
4905  * @return
4906  *   0 on success, negative value otherwise
4907  */
4908 static int
4909 flow_create_split_meter(struct rte_eth_dev *dev,
4910                         struct rte_flow *flow,
4911                         const struct rte_flow_attr *attr,
4912                         const struct rte_flow_item items[],
4913                         const struct rte_flow_action actions[],
4914                         struct mlx5_flow_split_info *flow_split_info,
4915                         struct rte_flow_error *error)
4916 {
4917         struct mlx5_priv *priv = dev->data->dev_private;
4918         struct rte_flow_action *sfx_actions = NULL;
4919         struct rte_flow_action *pre_actions = NULL;
4920         struct rte_flow_item *sfx_items = NULL;
4921         struct mlx5_flow *dev_flow = NULL;
4922         struct rte_flow_attr sfx_attr = *attr;
4923         uint32_t mtr = 0;
4924         uint32_t mtr_tag_id = 0;
4925         size_t act_size;
4926         size_t item_size;
4927         int actions_n = 0;
4928         int ret;
4929
4930         if (priv->mtr_en)
4931                 actions_n = flow_check_meter_action(actions, &mtr);
4932         if (mtr) {
4933                 /* The five prefix actions: meter, decap, encap, tag, end. */
4934                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4935                            sizeof(struct mlx5_rte_flow_action_set_tag);
4936                 /* tag, vlan, port id, end. */
4937 #define METER_SUFFIX_ITEM 4
4938                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4939                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4940                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4941                                           0, SOCKET_ID_ANY);
4942                 if (!sfx_actions)
4943                         return rte_flow_error_set(error, ENOMEM,
4944                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4945                                                   NULL, "no memory to split "
4946                                                   "meter flow");
4947                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4948                              act_size);
4949                 pre_actions = sfx_actions + actions_n;
4950                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4951                                                    actions, sfx_actions,
4952                                                    pre_actions);
4953                 if (!mtr_tag_id) {
4954                         ret = -rte_errno;
4955                         goto exit;
4956                 }
4957                 /* Add the prefix subflow. */
4958                 flow_split_info->prefix_mark = 0;
4959                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4960                                               attr, items, pre_actions,
4961                                               flow_split_info, error);
4962                 if (ret) {
4963                         ret = -rte_errno;
4964                         goto exit;
4965                 }
4966                 dev_flow->handle->split_flow_id = mtr_tag_id;
4967                 /* Setting the sfx group atrr. */
4968                 sfx_attr.group = sfx_attr.transfer ?
4969                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4970                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4971                 flow_split_info->prefix_layers =
4972                                 flow_get_prefix_layer_flags(dev_flow);
4973                 flow_split_info->prefix_mark = dev_flow->handle->mark;
4974         }
4975         /* Add the prefix subflow. */
4976         ret = flow_create_split_metadata(dev, flow,
4977                                          &sfx_attr, sfx_items ?
4978                                          sfx_items : items,
4979                                          sfx_actions ? sfx_actions : actions,
4980                                          flow_split_info, error);
4981 exit:
4982         if (sfx_actions)
4983                 mlx5_free(sfx_actions);
4984         return ret;
4985 }
4986
4987 /**
4988  * The splitting for sample feature.
4989  *
4990  * Once Sample action is detected in the action list, the flow actions should
4991  * be split into prefix sub flow and suffix sub flow.
4992  *
4993  * The original items remain in the prefix sub flow, all actions preceding the
4994  * sample action and the sample action itself will be copied to the prefix
4995  * sub flow, the actions following the sample action will be copied to the
4996  * suffix sub flow, Queue action always be located in the suffix sub flow.
4997  *
4998  * In order to make the packet from prefix sub flow matches with suffix sub
4999  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5000  * flow uses tag item with the unique flow id.
5001  *
5002  * @param dev
5003  *   Pointer to Ethernet device.
5004  * @param[in] flow
5005  *   Parent flow structure pointer.
5006  * @param[in] attr
5007  *   Flow rule attributes.
5008  * @param[in] items
5009  *   Pattern specification (list terminated by the END pattern item).
5010  * @param[in] actions
5011  *   Associated actions (list terminated by the END action).
5012  * @param[in] flow_split_info
5013  *   Pointer to flow split info structure.
5014  * @param[out] error
5015  *   Perform verbose error reporting if not NULL.
5016  * @return
5017  *   0 on success, negative value otherwise
5018  */
5019 static int
5020 flow_create_split_sample(struct rte_eth_dev *dev,
5021                          struct rte_flow *flow,
5022                          const struct rte_flow_attr *attr,
5023                          const struct rte_flow_item items[],
5024                          const struct rte_flow_action actions[],
5025                          struct mlx5_flow_split_info *flow_split_info,
5026                          struct rte_flow_error *error)
5027 {
5028         struct mlx5_priv *priv = dev->data->dev_private;
5029         struct rte_flow_action *sfx_actions = NULL;
5030         struct rte_flow_action *pre_actions = NULL;
5031         struct rte_flow_item *sfx_items = NULL;
5032         struct mlx5_flow *dev_flow = NULL;
5033         struct rte_flow_attr sfx_attr = *attr;
5034 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5035         struct mlx5_flow_dv_sample_resource *sample_res;
5036         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5037         struct mlx5_flow_tbl_resource *sfx_tbl;
5038 #endif
5039         size_t act_size;
5040         size_t item_size;
5041         uint32_t fdb_tx = 0;
5042         int32_t tag_id = 0;
5043         int actions_n = 0;
5044         int sample_action_pos;
5045         int qrss_action_pos;
5046         int ret = 0;
5047
5048         if (priv->sampler_en)
5049                 actions_n = flow_check_match_action(actions, attr,
5050                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5051                                         &sample_action_pos, &qrss_action_pos);
5052         if (actions_n) {
5053                 /* The prefix actions must includes sample, tag, end. */
5054                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5055                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5056                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5057                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5058                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5059                                           item_size), 0, SOCKET_ID_ANY);
5060                 if (!sfx_actions)
5061                         return rte_flow_error_set(error, ENOMEM,
5062                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5063                                                   NULL, "no memory to split "
5064                                                   "sample flow");
5065                 /* The representor_id is -1 for uplink. */
5066                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5067                 if (!fdb_tx)
5068                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5069                                         + act_size);
5070                 pre_actions = sfx_actions + actions_n;
5071                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5072                                                 actions, sfx_actions,
5073                                                 pre_actions, actions_n,
5074                                                 sample_action_pos,
5075                                                 qrss_action_pos, error);
5076                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5077                         ret = -rte_errno;
5078                         goto exit;
5079                 }
5080                 /* Add the prefix subflow. */
5081                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5082                                               items, pre_actions,
5083                                               flow_split_info, error);
5084                 if (ret) {
5085                         ret = -rte_errno;
5086                         goto exit;
5087                 }
5088                 dev_flow->handle->split_flow_id = tag_id;
5089 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5090                 /* Set the sfx group attr. */
5091                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5092                                         dev_flow->dv.sample_res;
5093                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5094                                         sample_res->normal_path_tbl;
5095                 sfx_tbl_data = container_of(sfx_tbl,
5096                                         struct mlx5_flow_tbl_data_entry, tbl);
5097                 sfx_attr.group = sfx_attr.transfer ?
5098                                         (sfx_tbl_data->table_id - 1) :
5099                                          sfx_tbl_data->table_id;
5100                 flow_split_info->prefix_layers =
5101                                 flow_get_prefix_layer_flags(dev_flow);
5102                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5103                 /* Suffix group level already be scaled with factor, set
5104                  * skip_scale to 1 to avoid scale again in translation.
5105                  */
5106                 flow_split_info->skip_scale = 1;
5107 #endif
5108         }
5109         /* Add the suffix subflow. */
5110         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5111                                       sfx_items ? sfx_items : items,
5112                                       sfx_actions ? sfx_actions : actions,
5113                                       flow_split_info, error);
5114 exit:
5115         if (sfx_actions)
5116                 mlx5_free(sfx_actions);
5117         return ret;
5118 }
5119
5120 /**
5121  * Split the flow to subflow set. The splitters might be linked
5122  * in the chain, like this:
5123  * flow_create_split_outer() calls:
5124  *   flow_create_split_meter() calls:
5125  *     flow_create_split_metadata(meter_subflow_0) calls:
5126  *       flow_create_split_inner(metadata_subflow_0)
5127  *       flow_create_split_inner(metadata_subflow_1)
5128  *       flow_create_split_inner(metadata_subflow_2)
5129  *     flow_create_split_metadata(meter_subflow_1) calls:
5130  *       flow_create_split_inner(metadata_subflow_0)
5131  *       flow_create_split_inner(metadata_subflow_1)
5132  *       flow_create_split_inner(metadata_subflow_2)
5133  *
5134  * This provide flexible way to add new levels of flow splitting.
5135  * The all of successfully created subflows are included to the
5136  * parent flow dev_flow list.
5137  *
5138  * @param dev
5139  *   Pointer to Ethernet device.
5140  * @param[in] flow
5141  *   Parent flow structure pointer.
5142  * @param[in] attr
5143  *   Flow rule attributes.
5144  * @param[in] items
5145  *   Pattern specification (list terminated by the END pattern item).
5146  * @param[in] actions
5147  *   Associated actions (list terminated by the END action).
5148  * @param[in] flow_split_info
5149  *   Pointer to flow split info structure.
5150  * @param[out] error
5151  *   Perform verbose error reporting if not NULL.
5152  * @return
5153  *   0 on success, negative value otherwise
5154  */
5155 static int
5156 flow_create_split_outer(struct rte_eth_dev *dev,
5157                         struct rte_flow *flow,
5158                         const struct rte_flow_attr *attr,
5159                         const struct rte_flow_item items[],
5160                         const struct rte_flow_action actions[],
5161                         struct mlx5_flow_split_info *flow_split_info,
5162                         struct rte_flow_error *error)
5163 {
5164         int ret;
5165
5166         ret = flow_create_split_sample(dev, flow, attr, items,
5167                                        actions, flow_split_info, error);
5168         MLX5_ASSERT(ret <= 0);
5169         return ret;
5170 }
5171
5172 static struct mlx5_flow_tunnel *
5173 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5174                       const struct rte_flow_attr *attr,
5175                       const struct rte_flow_item items[],
5176                       const struct rte_flow_action actions[])
5177 {
5178         struct mlx5_flow_tunnel *tunnel;
5179
5180 #pragma GCC diagnostic push
5181 #pragma GCC diagnostic ignored "-Wcast-qual"
5182         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5183                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5184         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5185                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5186         else
5187                 tunnel = NULL;
5188 #pragma GCC diagnostic pop
5189
5190         return tunnel;
5191 }
5192
5193 /**
5194  * Adjust flow RSS workspace if needed.
5195  *
5196  * @param wks
5197  *   Pointer to thread flow work space.
5198  * @param rss_desc
5199  *   Pointer to RSS descriptor.
5200  * @param[in] nrssq_num
5201  *   New RSS queue number.
5202  *
5203  * @return
5204  *   0 on success, -1 otherwise and rte_errno is set.
5205  */
5206 static int
5207 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
5208                           struct mlx5_flow_rss_desc *rss_desc,
5209                           uint32_t nrssq_num)
5210 {
5211         if (likely(nrssq_num <= wks->rssq_num))
5212                 return 0;
5213         rss_desc->queue = realloc(rss_desc->queue,
5214                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
5215         if (!rss_desc->queue) {
5216                 rte_errno = ENOMEM;
5217                 return -1;
5218         }
5219         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
5220         return 0;
5221 }
5222
5223 /**
5224  * Create a flow and add it to @p list.
5225  *
5226  * @param dev
5227  *   Pointer to Ethernet device.
5228  * @param list
5229  *   Pointer to a TAILQ flow list. If this parameter NULL,
5230  *   no list insertion occurred, flow is just created,
5231  *   this is caller's responsibility to track the
5232  *   created flow.
5233  * @param[in] attr
5234  *   Flow rule attributes.
5235  * @param[in] items
5236  *   Pattern specification (list terminated by the END pattern item).
5237  * @param[in] actions
5238  *   Associated actions (list terminated by the END action).
5239  * @param[in] external
5240  *   This flow rule is created by request external to PMD.
5241  * @param[out] error
5242  *   Perform verbose error reporting if not NULL.
5243  *
5244  * @return
5245  *   A flow index on success, 0 otherwise and rte_errno is set.
5246  */
5247 static uint32_t
5248 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5249                  const struct rte_flow_attr *attr,
5250                  const struct rte_flow_item items[],
5251                  const struct rte_flow_action original_actions[],
5252                  bool external, struct rte_flow_error *error)
5253 {
5254         struct mlx5_priv *priv = dev->data->dev_private;
5255         struct rte_flow *flow = NULL;
5256         struct mlx5_flow *dev_flow;
5257         const struct rte_flow_action_rss *rss;
5258         struct mlx5_translated_shared_action
5259                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5260         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5261         union {
5262                 struct mlx5_flow_expand_rss buf;
5263                 uint8_t buffer[2048];
5264         } expand_buffer;
5265         union {
5266                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5267                 uint8_t buffer[2048];
5268         } actions_rx;
5269         union {
5270                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5271                 uint8_t buffer[2048];
5272         } actions_hairpin_tx;
5273         union {
5274                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5275                 uint8_t buffer[2048];
5276         } items_tx;
5277         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5278         struct mlx5_flow_rss_desc *rss_desc;
5279         const struct rte_flow_action *p_actions_rx;
5280         uint32_t i;
5281         uint32_t idx = 0;
5282         int hairpin_flow;
5283         struct rte_flow_attr attr_tx = { .priority = 0 };
5284         const struct rte_flow_action *actions;
5285         struct rte_flow_action *translated_actions = NULL;
5286         struct mlx5_flow_tunnel *tunnel;
5287         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5288         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
5289         struct mlx5_flow_split_info flow_split_info = {
5290                 .external = !!external,
5291                 .skip_scale = 0,
5292                 .flow_idx = 0,
5293                 .prefix_mark = 0,
5294                 .prefix_layers = 0
5295         };
5296         int ret;
5297
5298         MLX5_ASSERT(wks);
5299         rss_desc = &wks->rss_desc;
5300         ret = flow_shared_actions_translate(dev, original_actions,
5301                                             shared_actions,
5302                                             &shared_actions_n,
5303                                             &translated_actions, error);
5304         if (ret < 0) {
5305                 MLX5_ASSERT(translated_actions == NULL);
5306                 return 0;
5307         }
5308         actions = translated_actions ? translated_actions : original_actions;
5309         p_actions_rx = actions;
5310         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5311         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
5312                                 external, hairpin_flow, error);
5313         if (ret < 0)
5314                 goto error_before_hairpin_split;
5315         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5316         if (!flow) {
5317                 rte_errno = ENOMEM;
5318                 goto error_before_hairpin_split;
5319         }
5320         if (hairpin_flow > 0) {
5321                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5322                         rte_errno = EINVAL;
5323                         goto error_before_hairpin_split;
5324                 }
5325                 flow_hairpin_split(dev, actions, actions_rx.actions,
5326                                    actions_hairpin_tx.actions, items_tx.items,
5327                                    idx);
5328                 p_actions_rx = actions_rx.actions;
5329         }
5330         flow_split_info.flow_idx = idx;
5331         flow->drv_type = flow_get_drv_type(dev, attr);
5332         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5333                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5334         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
5335         rss = flow_get_rss_action(p_actions_rx);
5336         if (rss) {
5337                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
5338                         return 0;
5339                 /*
5340                  * The following information is required by
5341                  * mlx5_flow_hashfields_adjust() in advance.
5342                  */
5343                 rss_desc->level = rss->level;
5344                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5345                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5346         }
5347         flow->dev_handles = 0;
5348         if (rss && rss->types) {
5349                 unsigned int graph_root;
5350
5351                 graph_root = find_graph_root(items, rss->level);
5352                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5353                                            items, rss->types,
5354                                            mlx5_support_expansion, graph_root);
5355                 MLX5_ASSERT(ret > 0 &&
5356                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5357         } else {
5358                 buf->entries = 1;
5359                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5360         }
5361         rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions,
5362                                                       shared_actions_n);
5363         for (i = 0; i < buf->entries; ++i) {
5364                 /* Initialize flow split data. */
5365                 flow_split_info.prefix_layers = 0;
5366                 flow_split_info.prefix_mark = 0;
5367                 flow_split_info.skip_scale = 0;
5368                 /*
5369                  * The splitter may create multiple dev_flows,
5370                  * depending on configuration. In the simplest
5371                  * case it just creates unmodified original flow.
5372                  */
5373                 ret = flow_create_split_outer(dev, flow, attr,
5374                                               buf->entry[i].pattern,
5375                                               p_actions_rx, &flow_split_info,
5376                                               error);
5377                 if (ret < 0)
5378                         goto error;
5379                 if (is_flow_tunnel_steer_rule(dev, attr,
5380                                               buf->entry[i].pattern,
5381                                               p_actions_rx)) {
5382                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5383                                                            p_actions_rx,
5384                                                            idx,
5385                                                            &default_miss_ctx,
5386                                                            error);
5387                         if (ret < 0) {
5388                                 mlx5_free(default_miss_ctx.queue);
5389                                 goto error;
5390                         }
5391                 }
5392         }
5393         /* Create the tx flow. */
5394         if (hairpin_flow) {
5395                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5396                 attr_tx.ingress = 0;
5397                 attr_tx.egress = 1;
5398                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5399                                          actions_hairpin_tx.actions,
5400                                          idx, error);
5401                 if (!dev_flow)
5402                         goto error;
5403                 dev_flow->flow = flow;
5404                 dev_flow->external = 0;
5405                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5406                               dev_flow->handle, next);
5407                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5408                                          items_tx.items,
5409                                          actions_hairpin_tx.actions, error);
5410                 if (ret < 0)
5411                         goto error;
5412         }
5413         /*
5414          * Update the metadata register copy table. If extensive
5415          * metadata feature is enabled and registers are supported
5416          * we might create the extra rte_flow for each unique
5417          * MARK/FLAG action ID.
5418          *
5419          * The table is updated for ingress Flows only, because
5420          * the egress Flows belong to the different device and
5421          * copy table should be updated in peer NIC Rx domain.
5422          */
5423         if (attr->ingress &&
5424             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5425                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5426                 if (ret)
5427                         goto error;
5428         }
5429         /*
5430          * If the flow is external (from application) OR device is started,
5431          * OR mreg discover, then apply immediately.
5432          */
5433         if (external || dev->data->dev_started ||
5434             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
5435              attr->priority == MLX5_FLOW_PRIO_RSVD)) {
5436                 ret = flow_drv_apply(dev, flow, error);
5437                 if (ret < 0)
5438                         goto error;
5439         }
5440         if (list) {
5441                 rte_spinlock_lock(&priv->flow_list_lock);
5442                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5443                              flow, next);
5444                 rte_spinlock_unlock(&priv->flow_list_lock);
5445         }
5446         flow_rxq_flags_set(dev, flow);
5447         rte_free(translated_actions);
5448         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5449         if (tunnel) {
5450                 flow->tunnel = 1;
5451                 flow->tunnel_id = tunnel->tunnel_id;
5452                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5453                 mlx5_free(default_miss_ctx.queue);
5454         }
5455         mlx5_flow_pop_thread_workspace();
5456         return idx;
5457 error:
5458         MLX5_ASSERT(flow);
5459         ret = rte_errno; /* Save rte_errno before cleanup. */
5460         flow_mreg_del_copy_action(dev, flow);
5461         flow_drv_destroy(dev, flow);
5462         if (rss_desc->shared_rss)
5463                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
5464                         mlx5_ipool_get
5465                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
5466                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
5467         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5468         rte_errno = ret; /* Restore rte_errno. */
5469         ret = rte_errno;
5470         rte_errno = ret;
5471         mlx5_flow_pop_thread_workspace();
5472 error_before_hairpin_split:
5473         rte_free(translated_actions);
5474         return 0;
5475 }
5476
5477 /**
5478  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5479  * incoming packets to table 1.
5480  *
5481  * Other flow rules, requested for group n, will be created in
5482  * e-switch table n+1.
5483  * Jump action to e-switch group n will be created to group n+1.
5484  *
5485  * Used when working in switchdev mode, to utilise advantages of table 1
5486  * and above.
5487  *
5488  * @param dev
5489  *   Pointer to Ethernet device.
5490  *
5491  * @return
5492  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5493  */
5494 struct rte_flow *
5495 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5496 {
5497         const struct rte_flow_attr attr = {
5498                 .group = 0,
5499                 .priority = 0,
5500                 .ingress = 1,
5501                 .egress = 0,
5502                 .transfer = 1,
5503         };
5504         const struct rte_flow_item pattern = {
5505                 .type = RTE_FLOW_ITEM_TYPE_END,
5506         };
5507         struct rte_flow_action_jump jump = {
5508                 .group = 1,
5509         };
5510         const struct rte_flow_action actions[] = {
5511                 {
5512                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5513                         .conf = &jump,
5514                 },
5515                 {
5516                         .type = RTE_FLOW_ACTION_TYPE_END,
5517                 },
5518         };
5519         struct mlx5_priv *priv = dev->data->dev_private;
5520         struct rte_flow_error error;
5521
5522         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5523                                                    &attr, &pattern,
5524                                                    actions, false, &error);
5525 }
5526
5527 /**
5528  * Validate a flow supported by the NIC.
5529  *
5530  * @see rte_flow_validate()
5531  * @see rte_flow_ops
5532  */
5533 int
5534 mlx5_flow_validate(struct rte_eth_dev *dev,
5535                    const struct rte_flow_attr *attr,
5536                    const struct rte_flow_item items[],
5537                    const struct rte_flow_action original_actions[],
5538                    struct rte_flow_error *error)
5539 {
5540         int hairpin_flow;
5541         struct mlx5_translated_shared_action
5542                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5543         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5544         const struct rte_flow_action *actions;
5545         struct rte_flow_action *translated_actions = NULL;
5546         int ret = flow_shared_actions_translate(dev, original_actions,
5547                                                 shared_actions,
5548                                                 &shared_actions_n,
5549                                                 &translated_actions, error);
5550
5551         if (ret)
5552                 return ret;
5553         actions = translated_actions ? translated_actions : original_actions;
5554         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5555         ret = flow_drv_validate(dev, attr, items, actions,
5556                                 true, hairpin_flow, error);
5557         rte_free(translated_actions);
5558         return ret;
5559 }
5560
5561 /**
5562  * Create a flow.
5563  *
5564  * @see rte_flow_create()
5565  * @see rte_flow_ops
5566  */
5567 struct rte_flow *
5568 mlx5_flow_create(struct rte_eth_dev *dev,
5569                  const struct rte_flow_attr *attr,
5570                  const struct rte_flow_item items[],
5571                  const struct rte_flow_action actions[],
5572                  struct rte_flow_error *error)
5573 {
5574         struct mlx5_priv *priv = dev->data->dev_private;
5575
5576         /*
5577          * If the device is not started yet, it is not allowed to created a
5578          * flow from application. PMD default flows and traffic control flows
5579          * are not affected.
5580          */
5581         if (unlikely(!dev->data->dev_started)) {
5582                 DRV_LOG(DEBUG, "port %u is not started when "
5583                         "inserting a flow", dev->data->port_id);
5584                 rte_flow_error_set(error, ENODEV,
5585                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5586                                    NULL,
5587                                    "port not started");
5588                 return NULL;
5589         }
5590
5591         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5592                                   attr, items, actions, true, error);
5593 }
5594
5595 /**
5596  * Destroy a flow in a list.
5597  *
5598  * @param dev
5599  *   Pointer to Ethernet device.
5600  * @param list
5601  *   Pointer to the Indexed flow list. If this parameter NULL,
5602  *   there is no flow removal from the list. Be noted that as
5603  *   flow is add to the indexed list, memory of the indexed
5604  *   list points to maybe changed as flow destroyed.
5605  * @param[in] flow_idx
5606  *   Index of flow to destroy.
5607  */
5608 static void
5609 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5610                   uint32_t flow_idx)
5611 {
5612         struct mlx5_priv *priv = dev->data->dev_private;
5613         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5614                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5615
5616         if (!flow)
5617                 return;
5618         /*
5619          * Update RX queue flags only if port is started, otherwise it is
5620          * already clean.
5621          */
5622         if (dev->data->dev_started)
5623                 flow_rxq_flags_trim(dev, flow);
5624         flow_drv_destroy(dev, flow);
5625         if (list) {
5626                 rte_spinlock_lock(&priv->flow_list_lock);
5627                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5628                              flow_idx, flow, next);
5629                 rte_spinlock_unlock(&priv->flow_list_lock);
5630         }
5631         if (flow->tunnel) {
5632                 struct mlx5_flow_tunnel *tunnel;
5633
5634                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5635                 RTE_VERIFY(tunnel);
5636                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5637                         mlx5_flow_tunnel_free(dev, tunnel);
5638         }
5639         flow_mreg_del_copy_action(dev, flow);
5640         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5641 }
5642
5643 /**
5644  * Destroy all flows.
5645  *
5646  * @param dev
5647  *   Pointer to Ethernet device.
5648  * @param list
5649  *   Pointer to the Indexed flow list.
5650  * @param active
5651  *   If flushing is called avtively.
5652  */
5653 void
5654 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5655 {
5656         uint32_t num_flushed = 0;
5657
5658         while (*list) {
5659                 flow_list_destroy(dev, list, *list);
5660                 num_flushed++;
5661         }
5662         if (active) {
5663                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5664                         dev->data->port_id, num_flushed);
5665         }
5666 }
5667
5668 /**
5669  * Stop all default actions for flows.
5670  *
5671  * @param dev
5672  *   Pointer to Ethernet device.
5673  */
5674 void
5675 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5676 {
5677         flow_mreg_del_default_copy_action(dev);
5678         flow_rxq_flags_clear(dev);
5679 }
5680
5681 /**
5682  * Start all default actions for flows.
5683  *
5684  * @param dev
5685  *   Pointer to Ethernet device.
5686  * @return
5687  *   0 on success, a negative errno value otherwise and rte_errno is set.
5688  */
5689 int
5690 mlx5_flow_start_default(struct rte_eth_dev *dev)
5691 {
5692         struct rte_flow_error error;
5693
5694         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5695         return flow_mreg_add_default_copy_action(dev, &error);
5696 }
5697
5698 /**
5699  * Release key of thread specific flow workspace data.
5700  */
5701 static void
5702 flow_release_workspace(void *data)
5703 {
5704         struct mlx5_flow_workspace *wks = data;
5705         struct mlx5_flow_workspace *next;
5706
5707         while (wks) {
5708                 next = wks->next;
5709                 free(wks->rss_desc.queue);
5710                 free(wks);
5711                 wks = next;
5712         }
5713 }
5714
5715 /**
5716  * Initialize key of thread specific flow workspace data.
5717  */
5718 static void
5719 flow_alloc_workspace(void)
5720 {
5721         if (pthread_key_create(&key_workspace, flow_release_workspace))
5722                 DRV_LOG(ERR, "Can't create flow workspace data thread key.");
5723 }
5724
5725 /**
5726  * Get thread specific current flow workspace.
5727  *
5728  * @return pointer to thread specific flow workspace data, NULL on error.
5729  */
5730 struct mlx5_flow_workspace*
5731 mlx5_flow_get_thread_workspace(void)
5732 {
5733         struct mlx5_flow_workspace *data;
5734
5735         data = pthread_getspecific(key_workspace);
5736         MLX5_ASSERT(data && data->inuse);
5737         if (!data || !data->inuse)
5738                 DRV_LOG(ERR, "flow workspace not initialized.");
5739         return data;
5740 }
5741
5742 /**
5743  * Allocate and init new flow workspace.
5744  *
5745  * @return pointer to flow workspace data, NULL on error.
5746  */
5747 static struct mlx5_flow_workspace*
5748 flow_alloc_thread_workspace(void)
5749 {
5750         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
5751
5752         if (!data) {
5753                 DRV_LOG(ERR, "Failed to allocate flow workspace "
5754                         "memory.");
5755                 return NULL;
5756         }
5757         data->rss_desc.queue = calloc(1,
5758                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
5759         if (!data->rss_desc.queue)
5760                 goto err;
5761         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
5762         return data;
5763 err:
5764         if (data->rss_desc.queue)
5765                 free(data->rss_desc.queue);
5766         free(data);
5767         return NULL;
5768 }
5769
5770 /**
5771  * Get new thread specific flow workspace.
5772  *
5773  * If current workspace inuse, create new one and set as current.
5774  *
5775  * @return pointer to thread specific flow workspace data, NULL on error.
5776  */
5777 static struct mlx5_flow_workspace*
5778 mlx5_flow_push_thread_workspace(void)
5779 {
5780         struct mlx5_flow_workspace *curr;
5781         struct mlx5_flow_workspace *data;
5782
5783         if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
5784                 DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
5785                 return NULL;
5786         }
5787         curr = pthread_getspecific(key_workspace);
5788         if (!curr) {
5789                 data = flow_alloc_thread_workspace();
5790                 if (!data)
5791                         return NULL;
5792         } else if (!curr->inuse) {
5793                 data = curr;
5794         } else if (curr->next) {
5795                 data = curr->next;
5796         } else {
5797                 data = flow_alloc_thread_workspace();
5798                 if (!data)
5799                         return NULL;
5800                 curr->next = data;
5801                 data->prev = curr;
5802         }
5803         data->inuse = 1;
5804         data->flow_idx = 0;
5805         /* Set as current workspace */
5806         if (pthread_setspecific(key_workspace, data))
5807                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5808         return data;
5809 }
5810
5811 /**
5812  * Close current thread specific flow workspace.
5813  *
5814  * If previous workspace available, set it as current.
5815  *
5816  * @return pointer to thread specific flow workspace data, NULL on error.
5817  */
5818 static void
5819 mlx5_flow_pop_thread_workspace(void)
5820 {
5821         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
5822
5823         if (!data)
5824                 return;
5825         if (!data->inuse) {
5826                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
5827                 return;
5828         }
5829         data->inuse = 0;
5830         if (!data->prev)
5831                 return;
5832         if (pthread_setspecific(key_workspace, data->prev))
5833                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5834 }
5835
5836 /**
5837  * Verify the flow list is empty
5838  *
5839  * @param dev
5840  *  Pointer to Ethernet device.
5841  *
5842  * @return the number of flows not released.
5843  */
5844 int
5845 mlx5_flow_verify(struct rte_eth_dev *dev)
5846 {
5847         struct mlx5_priv *priv = dev->data->dev_private;
5848         struct rte_flow *flow;
5849         uint32_t idx;
5850         int ret = 0;
5851
5852         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5853                       flow, next) {
5854                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5855                         dev->data->port_id, (void *)flow);
5856                 ++ret;
5857         }
5858         return ret;
5859 }
5860
5861 /**
5862  * Enable default hairpin egress flow.
5863  *
5864  * @param dev
5865  *   Pointer to Ethernet device.
5866  * @param queue
5867  *   The queue index.
5868  *
5869  * @return
5870  *   0 on success, a negative errno value otherwise and rte_errno is set.
5871  */
5872 int
5873 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5874                             uint32_t queue)
5875 {
5876         struct mlx5_priv *priv = dev->data->dev_private;
5877         const struct rte_flow_attr attr = {
5878                 .egress = 1,
5879                 .priority = 0,
5880         };
5881         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5882                 .queue = queue,
5883         };
5884         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5885                 .queue = UINT32_MAX,
5886         };
5887         struct rte_flow_item items[] = {
5888                 {
5889                         .type = (enum rte_flow_item_type)
5890                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5891                         .spec = &queue_spec,
5892                         .last = NULL,
5893                         .mask = &queue_mask,
5894                 },
5895                 {
5896                         .type = RTE_FLOW_ITEM_TYPE_END,
5897                 },
5898         };
5899         struct rte_flow_action_jump jump = {
5900                 .group = MLX5_HAIRPIN_TX_TABLE,
5901         };
5902         struct rte_flow_action actions[2];
5903         uint32_t flow_idx;
5904         struct rte_flow_error error;
5905
5906         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5907         actions[0].conf = &jump;
5908         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5909         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5910                                 &attr, items, actions, false, &error);
5911         if (!flow_idx) {
5912                 DRV_LOG(DEBUG,
5913                         "Failed to create ctrl flow: rte_errno(%d),"
5914                         " type(%d), message(%s)",
5915                         rte_errno, error.type,
5916                         error.message ? error.message : " (no stated reason)");
5917                 return -rte_errno;
5918         }
5919         return 0;
5920 }
5921
5922 /**
5923  * Enable a control flow configured from the control plane.
5924  *
5925  * @param dev
5926  *   Pointer to Ethernet device.
5927  * @param eth_spec
5928  *   An Ethernet flow spec to apply.
5929  * @param eth_mask
5930  *   An Ethernet flow mask to apply.
5931  * @param vlan_spec
5932  *   A VLAN flow spec to apply.
5933  * @param vlan_mask
5934  *   A VLAN flow mask to apply.
5935  *
5936  * @return
5937  *   0 on success, a negative errno value otherwise and rte_errno is set.
5938  */
5939 int
5940 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5941                     struct rte_flow_item_eth *eth_spec,
5942                     struct rte_flow_item_eth *eth_mask,
5943                     struct rte_flow_item_vlan *vlan_spec,
5944                     struct rte_flow_item_vlan *vlan_mask)
5945 {
5946         struct mlx5_priv *priv = dev->data->dev_private;
5947         const struct rte_flow_attr attr = {
5948                 .ingress = 1,
5949                 .priority = MLX5_FLOW_PRIO_RSVD,
5950         };
5951         struct rte_flow_item items[] = {
5952                 {
5953                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5954                         .spec = eth_spec,
5955                         .last = NULL,
5956                         .mask = eth_mask,
5957                 },
5958                 {
5959                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5960                                               RTE_FLOW_ITEM_TYPE_END,
5961                         .spec = vlan_spec,
5962                         .last = NULL,
5963                         .mask = vlan_mask,
5964                 },
5965                 {
5966                         .type = RTE_FLOW_ITEM_TYPE_END,
5967                 },
5968         };
5969         uint16_t queue[priv->reta_idx_n];
5970         struct rte_flow_action_rss action_rss = {
5971                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5972                 .level = 0,
5973                 .types = priv->rss_conf.rss_hf,
5974                 .key_len = priv->rss_conf.rss_key_len,
5975                 .queue_num = priv->reta_idx_n,
5976                 .key = priv->rss_conf.rss_key,
5977                 .queue = queue,
5978         };
5979         struct rte_flow_action actions[] = {
5980                 {
5981                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5982                         .conf = &action_rss,
5983                 },
5984                 {
5985                         .type = RTE_FLOW_ACTION_TYPE_END,
5986                 },
5987         };
5988         uint32_t flow_idx;
5989         struct rte_flow_error error;
5990         unsigned int i;
5991
5992         if (!priv->reta_idx_n || !priv->rxqs_n) {
5993                 return 0;
5994         }
5995         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5996                 action_rss.types = 0;
5997         for (i = 0; i != priv->reta_idx_n; ++i)
5998                 queue[i] = (*priv->reta_idx)[i];
5999         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6000                                 &attr, items, actions, false, &error);
6001         if (!flow_idx)
6002                 return -rte_errno;
6003         return 0;
6004 }
6005
6006 /**
6007  * Enable a flow control configured from the control plane.
6008  *
6009  * @param dev
6010  *   Pointer to Ethernet device.
6011  * @param eth_spec
6012  *   An Ethernet flow spec to apply.
6013  * @param eth_mask
6014  *   An Ethernet flow mask to apply.
6015  *
6016  * @return
6017  *   0 on success, a negative errno value otherwise and rte_errno is set.
6018  */
6019 int
6020 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6021                struct rte_flow_item_eth *eth_spec,
6022                struct rte_flow_item_eth *eth_mask)
6023 {
6024         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6025 }
6026
6027 /**
6028  * Create default miss flow rule matching lacp traffic
6029  *
6030  * @param dev
6031  *   Pointer to Ethernet device.
6032  * @param eth_spec
6033  *   An Ethernet flow spec to apply.
6034  *
6035  * @return
6036  *   0 on success, a negative errno value otherwise and rte_errno is set.
6037  */
6038 int
6039 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6040 {
6041         struct mlx5_priv *priv = dev->data->dev_private;
6042         /*
6043          * The LACP matching is done by only using ether type since using
6044          * a multicast dst mac causes kernel to give low priority to this flow.
6045          */
6046         static const struct rte_flow_item_eth lacp_spec = {
6047                 .type = RTE_BE16(0x8809),
6048         };
6049         static const struct rte_flow_item_eth lacp_mask = {
6050                 .type = 0xffff,
6051         };
6052         const struct rte_flow_attr attr = {
6053                 .ingress = 1,
6054         };
6055         struct rte_flow_item items[] = {
6056                 {
6057                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6058                         .spec = &lacp_spec,
6059                         .mask = &lacp_mask,
6060                 },
6061                 {
6062                         .type = RTE_FLOW_ITEM_TYPE_END,
6063                 },
6064         };
6065         struct rte_flow_action actions[] = {
6066                 {
6067                         .type = (enum rte_flow_action_type)
6068                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6069                 },
6070                 {
6071                         .type = RTE_FLOW_ACTION_TYPE_END,
6072                 },
6073         };
6074         struct rte_flow_error error;
6075         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6076                                 &attr, items, actions, false, &error);
6077
6078         if (!flow_idx)
6079                 return -rte_errno;
6080         return 0;
6081 }
6082
6083 /**
6084  * Destroy a flow.
6085  *
6086  * @see rte_flow_destroy()
6087  * @see rte_flow_ops
6088  */
6089 int
6090 mlx5_flow_destroy(struct rte_eth_dev *dev,
6091                   struct rte_flow *flow,
6092                   struct rte_flow_error *error __rte_unused)
6093 {
6094         struct mlx5_priv *priv = dev->data->dev_private;
6095
6096         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6097         return 0;
6098 }
6099
6100 /**
6101  * Destroy all flows.
6102  *
6103  * @see rte_flow_flush()
6104  * @see rte_flow_ops
6105  */
6106 int
6107 mlx5_flow_flush(struct rte_eth_dev *dev,
6108                 struct rte_flow_error *error __rte_unused)
6109 {
6110         struct mlx5_priv *priv = dev->data->dev_private;
6111
6112         mlx5_flow_list_flush(dev, &priv->flows, false);
6113         return 0;
6114 }
6115
6116 /**
6117  * Isolated mode.
6118  *
6119  * @see rte_flow_isolate()
6120  * @see rte_flow_ops
6121  */
6122 int
6123 mlx5_flow_isolate(struct rte_eth_dev *dev,
6124                   int enable,
6125                   struct rte_flow_error *error)
6126 {
6127         struct mlx5_priv *priv = dev->data->dev_private;
6128
6129         if (dev->data->dev_started) {
6130                 rte_flow_error_set(error, EBUSY,
6131                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6132                                    NULL,
6133                                    "port must be stopped first");
6134                 return -rte_errno;
6135         }
6136         priv->isolated = !!enable;
6137         if (enable)
6138                 dev->dev_ops = &mlx5_dev_ops_isolate;
6139         else
6140                 dev->dev_ops = &mlx5_dev_ops;
6141
6142         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6143         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6144
6145         return 0;
6146 }
6147
6148 /**
6149  * Query a flow.
6150  *
6151  * @see rte_flow_query()
6152  * @see rte_flow_ops
6153  */
6154 static int
6155 flow_drv_query(struct rte_eth_dev *dev,
6156                uint32_t flow_idx,
6157                const struct rte_flow_action *actions,
6158                void *data,
6159                struct rte_flow_error *error)
6160 {
6161         struct mlx5_priv *priv = dev->data->dev_private;
6162         const struct mlx5_flow_driver_ops *fops;
6163         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6164                                                [MLX5_IPOOL_RTE_FLOW],
6165                                                flow_idx);
6166         enum mlx5_flow_drv_type ftype;
6167
6168         if (!flow) {
6169                 return rte_flow_error_set(error, ENOENT,
6170                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6171                           NULL,
6172                           "invalid flow handle");
6173         }
6174         ftype = flow->drv_type;
6175         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6176         fops = flow_get_drv_ops(ftype);
6177
6178         return fops->query(dev, flow, actions, data, error);
6179 }
6180
6181 /**
6182  * Query a flow.
6183  *
6184  * @see rte_flow_query()
6185  * @see rte_flow_ops
6186  */
6187 int
6188 mlx5_flow_query(struct rte_eth_dev *dev,
6189                 struct rte_flow *flow,
6190                 const struct rte_flow_action *actions,
6191                 void *data,
6192                 struct rte_flow_error *error)
6193 {
6194         int ret;
6195
6196         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6197                              error);
6198         if (ret < 0)
6199                 return ret;
6200         return 0;
6201 }
6202
6203 /**
6204  * Manage filter operations.
6205  *
6206  * @param dev
6207  *   Pointer to Ethernet device structure.
6208  * @param filter_type
6209  *   Filter type.
6210  * @param filter_op
6211  *   Operation to perform.
6212  * @param arg
6213  *   Pointer to operation-specific structure.
6214  *
6215  * @return
6216  *   0 on success, a negative errno value otherwise and rte_errno is set.
6217  */
6218 int
6219 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6220                      enum rte_filter_type filter_type,
6221                      enum rte_filter_op filter_op,
6222                      void *arg)
6223 {
6224         switch (filter_type) {
6225         case RTE_ETH_FILTER_GENERIC:
6226                 if (filter_op != RTE_ETH_FILTER_GET) {
6227                         rte_errno = EINVAL;
6228                         return -rte_errno;
6229                 }
6230                 *(const void **)arg = &mlx5_flow_ops;
6231                 return 0;
6232         default:
6233                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6234                         dev->data->port_id, filter_type);
6235                 rte_errno = ENOTSUP;
6236                 return -rte_errno;
6237         }
6238         return 0;
6239 }
6240
6241 /**
6242  * Create the needed meter and suffix tables.
6243  *
6244  * @param[in] dev
6245  *   Pointer to Ethernet device.
6246  * @param[in] fm
6247  *   Pointer to the flow meter.
6248  *
6249  * @return
6250  *   Pointer to table set on success, NULL otherwise.
6251  */
6252 struct mlx5_meter_domains_infos *
6253 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6254                           const struct mlx5_flow_meter *fm)
6255 {
6256         const struct mlx5_flow_driver_ops *fops;
6257
6258         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6259         return fops->create_mtr_tbls(dev, fm);
6260 }
6261
6262 /**
6263  * Destroy the meter table set.
6264  *
6265  * @param[in] dev
6266  *   Pointer to Ethernet device.
6267  * @param[in] tbl
6268  *   Pointer to the meter table set.
6269  *
6270  * @return
6271  *   0 on success.
6272  */
6273 int
6274 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6275                            struct mlx5_meter_domains_infos *tbls)
6276 {
6277         const struct mlx5_flow_driver_ops *fops;
6278
6279         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6280         return fops->destroy_mtr_tbls(dev, tbls);
6281 }
6282
6283 /**
6284  * Create policer rules.
6285  *
6286  * @param[in] dev
6287  *   Pointer to Ethernet device.
6288  * @param[in] fm
6289  *   Pointer to flow meter structure.
6290  * @param[in] attr
6291  *   Pointer to flow attributes.
6292  *
6293  * @return
6294  *   0 on success, -1 otherwise.
6295  */
6296 int
6297 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6298                                struct mlx5_flow_meter *fm,
6299                                const struct rte_flow_attr *attr)
6300 {
6301         const struct mlx5_flow_driver_ops *fops;
6302
6303         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6304         return fops->create_policer_rules(dev, fm, attr);
6305 }
6306
6307 /**
6308  * Destroy policer rules.
6309  *
6310  * @param[in] fm
6311  *   Pointer to flow meter structure.
6312  * @param[in] attr
6313  *   Pointer to flow attributes.
6314  *
6315  * @return
6316  *   0 on success, -1 otherwise.
6317  */
6318 int
6319 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6320                                 struct mlx5_flow_meter *fm,
6321                                 const struct rte_flow_attr *attr)
6322 {
6323         const struct mlx5_flow_driver_ops *fops;
6324
6325         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6326         return fops->destroy_policer_rules(dev, fm, attr);
6327 }
6328
6329 /**
6330  * Allocate a counter.
6331  *
6332  * @param[in] dev
6333  *   Pointer to Ethernet device structure.
6334  *
6335  * @return
6336  *   Index to allocated counter  on success, 0 otherwise.
6337  */
6338 uint32_t
6339 mlx5_counter_alloc(struct rte_eth_dev *dev)
6340 {
6341         const struct mlx5_flow_driver_ops *fops;
6342         struct rte_flow_attr attr = { .transfer = 0 };
6343
6344         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6345                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6346                 return fops->counter_alloc(dev);
6347         }
6348         DRV_LOG(ERR,
6349                 "port %u counter allocate is not supported.",
6350                  dev->data->port_id);
6351         return 0;
6352 }
6353
6354 /**
6355  * Free a counter.
6356  *
6357  * @param[in] dev
6358  *   Pointer to Ethernet device structure.
6359  * @param[in] cnt
6360  *   Index to counter to be free.
6361  */
6362 void
6363 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6364 {
6365         const struct mlx5_flow_driver_ops *fops;
6366         struct rte_flow_attr attr = { .transfer = 0 };
6367
6368         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6369                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6370                 fops->counter_free(dev, cnt);
6371                 return;
6372         }
6373         DRV_LOG(ERR,
6374                 "port %u counter free is not supported.",
6375                  dev->data->port_id);
6376 }
6377
6378 /**
6379  * Query counter statistics.
6380  *
6381  * @param[in] dev
6382  *   Pointer to Ethernet device structure.
6383  * @param[in] cnt
6384  *   Index to counter to query.
6385  * @param[in] clear
6386  *   Set to clear counter statistics.
6387  * @param[out] pkts
6388  *   The counter hits packets number to save.
6389  * @param[out] bytes
6390  *   The counter hits bytes number to save.
6391  *
6392  * @return
6393  *   0 on success, a negative errno value otherwise.
6394  */
6395 int
6396 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6397                    bool clear, uint64_t *pkts, uint64_t *bytes)
6398 {
6399         const struct mlx5_flow_driver_ops *fops;
6400         struct rte_flow_attr attr = { .transfer = 0 };
6401
6402         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6403                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6404                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6405         }
6406         DRV_LOG(ERR,
6407                 "port %u counter query is not supported.",
6408                  dev->data->port_id);
6409         return -ENOTSUP;
6410 }
6411
6412 /**
6413  * Allocate a new memory for the counter values wrapped by all the needed
6414  * management.
6415  *
6416  * @param[in] sh
6417  *   Pointer to mlx5_dev_ctx_shared object.
6418  *
6419  * @return
6420  *   0 on success, a negative errno value otherwise.
6421  */
6422 static int
6423 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
6424 {
6425         struct mlx5_devx_mkey_attr mkey_attr;
6426         struct mlx5_counter_stats_mem_mng *mem_mng;
6427         volatile struct flow_counter_stats *raw_data;
6428         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
6429         int size = (sizeof(struct flow_counter_stats) *
6430                         MLX5_COUNTERS_PER_POOL +
6431                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
6432                         sizeof(struct mlx5_counter_stats_mem_mng);
6433         size_t pgsize = rte_mem_page_size();
6434         uint8_t *mem;
6435         int i;
6436
6437         if (pgsize == (size_t)-1) {
6438                 DRV_LOG(ERR, "Failed to get mem page size");
6439                 rte_errno = ENOMEM;
6440                 return -ENOMEM;
6441         }
6442         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
6443         if (!mem) {
6444                 rte_errno = ENOMEM;
6445                 return -ENOMEM;
6446         }
6447         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
6448         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
6449         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
6450                                                  IBV_ACCESS_LOCAL_WRITE);
6451         if (!mem_mng->umem) {
6452                 rte_errno = errno;
6453                 mlx5_free(mem);
6454                 return -rte_errno;
6455         }
6456         mkey_attr.addr = (uintptr_t)mem;
6457         mkey_attr.size = size;
6458         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
6459         mkey_attr.pd = sh->pdn;
6460         mkey_attr.log_entity_size = 0;
6461         mkey_attr.pg_access = 0;
6462         mkey_attr.klm_array = NULL;
6463         mkey_attr.klm_num = 0;
6464         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
6465         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
6466         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
6467         if (!mem_mng->dm) {
6468                 mlx5_os_umem_dereg(mem_mng->umem);
6469                 rte_errno = errno;
6470                 mlx5_free(mem);
6471                 return -rte_errno;
6472         }
6473         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
6474         raw_data = (volatile struct flow_counter_stats *)mem;
6475         for (i = 0; i < raws_n; ++i) {
6476                 mem_mng->raws[i].mem_mng = mem_mng;
6477                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
6478         }
6479         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
6480                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
6481                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
6482                                  next);
6483         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
6484         sh->cmng.mem_mng = mem_mng;
6485         return 0;
6486 }
6487
6488 /**
6489  * Set the statistic memory to the new counter pool.
6490  *
6491  * @param[in] sh
6492  *   Pointer to mlx5_dev_ctx_shared object.
6493  * @param[in] pool
6494  *   Pointer to the pool to set the statistic memory.
6495  *
6496  * @return
6497  *   0 on success, a negative errno value otherwise.
6498  */
6499 static int
6500 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
6501                                struct mlx5_flow_counter_pool *pool)
6502 {
6503         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6504         /* Resize statistic memory once used out. */
6505         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
6506             mlx5_flow_create_counter_stat_mem_mng(sh)) {
6507                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
6508                 return -1;
6509         }
6510         rte_spinlock_lock(&pool->sl);
6511         pool->raw = cmng->mem_mng->raws + pool->index %
6512                     MLX5_CNT_CONTAINER_RESIZE;
6513         rte_spinlock_unlock(&pool->sl);
6514         pool->raw_hw = NULL;
6515         return 0;
6516 }
6517
6518 #define MLX5_POOL_QUERY_FREQ_US 1000000
6519
6520 /**
6521  * Set the periodic procedure for triggering asynchronous batch queries for all
6522  * the counter pools.
6523  *
6524  * @param[in] sh
6525  *   Pointer to mlx5_dev_ctx_shared object.
6526  */
6527 void
6528 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6529 {
6530         uint32_t pools_n, us;
6531
6532         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
6533         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6534         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6535         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6536                 sh->cmng.query_thread_on = 0;
6537                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6538         } else {
6539                 sh->cmng.query_thread_on = 1;
6540         }
6541 }
6542
6543 /**
6544  * The periodic procedure for triggering asynchronous batch queries for all the
6545  * counter pools. This function is probably called by the host thread.
6546  *
6547  * @param[in] arg
6548  *   The parameter for the alarm process.
6549  */
6550 void
6551 mlx5_flow_query_alarm(void *arg)
6552 {
6553         struct mlx5_dev_ctx_shared *sh = arg;
6554         int ret;
6555         uint16_t pool_index = sh->cmng.pool_index;
6556         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6557         struct mlx5_flow_counter_pool *pool;
6558         uint16_t n_valid;
6559
6560         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6561                 goto set_alarm;
6562         rte_spinlock_lock(&cmng->pool_update_sl);
6563         pool = cmng->pools[pool_index];
6564         n_valid = cmng->n_valid;
6565         rte_spinlock_unlock(&cmng->pool_update_sl);
6566         /* Set the statistic memory to the new created pool. */
6567         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
6568                 goto set_alarm;
6569         if (pool->raw_hw)
6570                 /* There is a pool query in progress. */
6571                 goto set_alarm;
6572         pool->raw_hw =
6573                 LIST_FIRST(&sh->cmng.free_stat_raws);
6574         if (!pool->raw_hw)
6575                 /* No free counter statistics raw memory. */
6576                 goto set_alarm;
6577         /*
6578          * Identify the counters released between query trigger and query
6579          * handle more efficiently. The counter released in this gap period
6580          * should wait for a new round of query as the new arrived packets
6581          * will not be taken into account.
6582          */
6583         pool->query_gen++;
6584         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
6585                                                MLX5_COUNTERS_PER_POOL,
6586                                                NULL, NULL,
6587                                                pool->raw_hw->mem_mng->dm->id,
6588                                                (void *)(uintptr_t)
6589                                                pool->raw_hw->data,
6590                                                sh->devx_comp,
6591                                                (uint64_t)(uintptr_t)pool);
6592         if (ret) {
6593                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6594                         " %d", pool->min_dcs->id);
6595                 pool->raw_hw = NULL;
6596                 goto set_alarm;
6597         }
6598         LIST_REMOVE(pool->raw_hw, next);
6599         sh->cmng.pending_queries++;
6600         pool_index++;
6601         if (pool_index >= n_valid)
6602                 pool_index = 0;
6603 set_alarm:
6604         sh->cmng.pool_index = pool_index;
6605         mlx5_set_query_alarm(sh);
6606 }
6607
6608 /**
6609  * Check and callback event for new aged flow in the counter pool
6610  *
6611  * @param[in] sh
6612  *   Pointer to mlx5_dev_ctx_shared object.
6613  * @param[in] pool
6614  *   Pointer to Current counter pool.
6615  */
6616 static void
6617 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6618                    struct mlx5_flow_counter_pool *pool)
6619 {
6620         struct mlx5_priv *priv;
6621         struct mlx5_flow_counter *cnt;
6622         struct mlx5_age_info *age_info;
6623         struct mlx5_age_param *age_param;
6624         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6625         struct mlx5_counter_stats_raw *prev = pool->raw;
6626         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
6627         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
6628         uint16_t expected = AGE_CANDIDATE;
6629         uint32_t i;
6630
6631         pool->time_of_last_age_check = curr_time;
6632         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6633                 cnt = MLX5_POOL_GET_CNT(pool, i);
6634                 age_param = MLX5_CNT_TO_AGE(cnt);
6635                 if (__atomic_load_n(&age_param->state,
6636                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
6637                         continue;
6638                 if (cur->data[i].hits != prev->data[i].hits) {
6639                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
6640                                          __ATOMIC_RELAXED);
6641                         continue;
6642                 }
6643                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
6644                                        time_delta,
6645                                        __ATOMIC_RELAXED) <= age_param->timeout)
6646                         continue;
6647                 /**
6648                  * Hold the lock first, or if between the
6649                  * state AGE_TMOUT and tailq operation the
6650                  * release happened, the release procedure
6651                  * may delete a non-existent tailq node.
6652                  */
6653                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6654                 age_info = GET_PORT_AGE_INFO(priv);
6655                 rte_spinlock_lock(&age_info->aged_sl);
6656                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
6657                                                 AGE_TMOUT, false,
6658                                                 __ATOMIC_RELAXED,
6659                                                 __ATOMIC_RELAXED)) {
6660                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6661                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6662                 }
6663                 rte_spinlock_unlock(&age_info->aged_sl);
6664         }
6665         mlx5_age_event_prepare(sh);
6666 }
6667
6668 /**
6669  * Handler for the HW respond about ready values from an asynchronous batch
6670  * query. This function is probably called by the host thread.
6671  *
6672  * @param[in] sh
6673  *   The pointer to the shared device context.
6674  * @param[in] async_id
6675  *   The Devx async ID.
6676  * @param[in] status
6677  *   The status of the completion.
6678  */
6679 void
6680 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6681                                   uint64_t async_id, int status)
6682 {
6683         struct mlx5_flow_counter_pool *pool =
6684                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6685         struct mlx5_counter_stats_raw *raw_to_free;
6686         uint8_t query_gen = pool->query_gen ^ 1;
6687         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6688         enum mlx5_counter_type cnt_type =
6689                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6690                                 MLX5_COUNTER_TYPE_ORIGIN;
6691
6692         if (unlikely(status)) {
6693                 raw_to_free = pool->raw_hw;
6694         } else {
6695                 raw_to_free = pool->raw;
6696                 if (pool->is_aged)
6697                         mlx5_flow_aging_check(sh, pool);
6698                 rte_spinlock_lock(&pool->sl);
6699                 pool->raw = pool->raw_hw;
6700                 rte_spinlock_unlock(&pool->sl);
6701                 /* Be sure the new raw counters data is updated in memory. */
6702                 rte_io_wmb();
6703                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6704                         rte_spinlock_lock(&cmng->csl[cnt_type]);
6705                         TAILQ_CONCAT(&cmng->counters[cnt_type],
6706                                      &pool->counters[query_gen], next);
6707                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6708                 }
6709         }
6710         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6711         pool->raw_hw = NULL;
6712         sh->cmng.pending_queries--;
6713 }
6714
6715 static int
6716 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
6717                     const struct flow_grp_info *grp_info,
6718                     struct rte_flow_error *error)
6719 {
6720         if (grp_info->transfer && grp_info->external &&
6721             grp_info->fdb_def_rule) {
6722                 if (group == UINT32_MAX)
6723                         return rte_flow_error_set
6724                                                 (error, EINVAL,
6725                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6726                                                  NULL,
6727                                                  "group index not supported");
6728                 *table = group + 1;
6729         } else {
6730                 *table = group;
6731         }
6732         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
6733         return 0;
6734 }
6735
6736 /**
6737  * Translate the rte_flow group index to HW table value.
6738  *
6739  * If tunnel offload is disabled, all group ids converted to flow table
6740  * id using the standard method.
6741  * If tunnel offload is enabled, group id can be converted using the
6742  * standard or tunnel conversion method. Group conversion method
6743  * selection depends on flags in `grp_info` parameter:
6744  * - Internal (grp_info.external == 0) groups conversion uses the
6745  *   standard method.
6746  * - Group ids in JUMP action converted with the tunnel conversion.
6747  * - Group id in rule attribute conversion depends on a rule type and
6748  *   group id value:
6749  *   ** non zero group attributes converted with the tunnel method
6750  *   ** zero group attribute in non-tunnel rule is converted using the
6751  *      standard method - there's only one root table
6752  *   ** zero group attribute in steer tunnel rule is converted with the
6753  *      standard method - single root table
6754  *   ** zero group attribute in match tunnel rule is a special OvS
6755  *      case: that value is used for portability reasons. That group
6756  *      id is converted with the tunnel conversion method.
6757  *
6758  * @param[in] dev
6759  *   Port device
6760  * @param[in] tunnel
6761  *   PMD tunnel offload object
6762  * @param[in] group
6763  *   rte_flow group index value.
6764  * @param[out] table
6765  *   HW table value.
6766  * @param[in] grp_info
6767  *   flags used for conversion
6768  * @param[out] error
6769  *   Pointer to error structure.
6770  *
6771  * @return
6772  *   0 on success, a negative errno value otherwise and rte_errno is set.
6773  */
6774 int
6775 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
6776                          const struct mlx5_flow_tunnel *tunnel,
6777                          uint32_t group, uint32_t *table,
6778                          const struct flow_grp_info *grp_info,
6779                          struct rte_flow_error *error)
6780 {
6781         int ret;
6782         bool standard_translation;
6783
6784         if (!grp_info->skip_scale && grp_info->external &&
6785             group < MLX5_MAX_TABLES_EXTERNAL)
6786                 group *= MLX5_FLOW_TABLE_FACTOR;
6787         if (is_tunnel_offload_active(dev)) {
6788                 standard_translation = !grp_info->external ||
6789                                         grp_info->std_tbl_fix;
6790         } else {
6791                 standard_translation = true;
6792         }
6793         DRV_LOG(DEBUG,
6794                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
6795                 dev->data->port_id, group, grp_info->transfer,
6796                 grp_info->external, grp_info->fdb_def_rule,
6797                 standard_translation ? "STANDARD" : "TUNNEL");
6798         if (standard_translation)
6799                 ret = flow_group_to_table(dev->data->port_id, group, table,
6800                                           grp_info, error);
6801         else
6802                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
6803                                                       table, error);
6804
6805         return ret;
6806 }
6807
6808 /**
6809  * Discover availability of metadata reg_c's.
6810  *
6811  * Iteratively use test flows to check availability.
6812  *
6813  * @param[in] dev
6814  *   Pointer to the Ethernet device structure.
6815  *
6816  * @return
6817  *   0 on success, a negative errno value otherwise and rte_errno is set.
6818  */
6819 int
6820 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6821 {
6822         struct mlx5_priv *priv = dev->data->dev_private;
6823         struct mlx5_dev_config *config = &priv->config;
6824         enum modify_reg idx;
6825         int n = 0;
6826
6827         /* reg_c[0] and reg_c[1] are reserved. */
6828         config->flow_mreg_c[n++] = REG_C_0;
6829         config->flow_mreg_c[n++] = REG_C_1;
6830         /* Discover availability of other reg_c's. */
6831         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6832                 struct rte_flow_attr attr = {
6833                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6834                         .priority = MLX5_FLOW_PRIO_RSVD,
6835                         .ingress = 1,
6836                 };
6837                 struct rte_flow_item items[] = {
6838                         [0] = {
6839                                 .type = RTE_FLOW_ITEM_TYPE_END,
6840                         },
6841                 };
6842                 struct rte_flow_action actions[] = {
6843                         [0] = {
6844                                 .type = (enum rte_flow_action_type)
6845                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6846                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6847                                         .src = REG_C_1,
6848                                         .dst = idx,
6849                                 },
6850                         },
6851                         [1] = {
6852                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6853                                 .conf = &(struct rte_flow_action_jump){
6854                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6855                                 },
6856                         },
6857                         [2] = {
6858                                 .type = RTE_FLOW_ACTION_TYPE_END,
6859                         },
6860                 };
6861                 uint32_t flow_idx;
6862                 struct rte_flow *flow;
6863                 struct rte_flow_error error;
6864
6865                 if (!config->dv_flow_en)
6866                         break;
6867                 /* Create internal flow, validation skips copy action. */
6868                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6869                                             actions, false, &error);
6870                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6871                                       flow_idx);
6872                 if (!flow)
6873                         continue;
6874                 config->flow_mreg_c[n++] = idx;
6875                 flow_list_destroy(dev, NULL, flow_idx);
6876         }
6877         for (; n < MLX5_MREG_C_NUM; ++n)
6878                 config->flow_mreg_c[n] = REG_NON;
6879         return 0;
6880 }
6881
6882 /**
6883  * Dump flow raw hw data to file
6884  *
6885  * @param[in] dev
6886  *    The pointer to Ethernet device.
6887  * @param[in] file
6888  *   A pointer to a file for output.
6889  * @param[out] error
6890  *   Perform verbose error reporting if not NULL. PMDs initialize this
6891  *   structure in case of error only.
6892  * @return
6893  *   0 on success, a nagative value otherwise.
6894  */
6895 int
6896 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6897                    FILE *file,
6898                    struct rte_flow_error *error __rte_unused)
6899 {
6900         struct mlx5_priv *priv = dev->data->dev_private;
6901         struct mlx5_dev_ctx_shared *sh = priv->sh;
6902
6903         if (!priv->config.dv_flow_en) {
6904                 if (fputs("device dv flow disabled\n", file) <= 0)
6905                         return -errno;
6906                 return -ENOTSUP;
6907         }
6908         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6909                                        sh->tx_domain, file);
6910 }
6911
6912 /**
6913  * Get aged-out flows.
6914  *
6915  * @param[in] dev
6916  *   Pointer to the Ethernet device structure.
6917  * @param[in] context
6918  *   The address of an array of pointers to the aged-out flows contexts.
6919  * @param[in] nb_countexts
6920  *   The length of context array pointers.
6921  * @param[out] error
6922  *   Perform verbose error reporting if not NULL. Initialized in case of
6923  *   error only.
6924  *
6925  * @return
6926  *   how many contexts get in success, otherwise negative errno value.
6927  *   if nb_contexts is 0, return the amount of all aged contexts.
6928  *   if nb_contexts is not 0 , return the amount of aged flows reported
6929  *   in the context array.
6930  */
6931 int
6932 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6933                         uint32_t nb_contexts, struct rte_flow_error *error)
6934 {
6935         const struct mlx5_flow_driver_ops *fops;
6936         struct rte_flow_attr attr = { .transfer = 0 };
6937
6938         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6939                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6940                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6941                                                     error);
6942         }
6943         DRV_LOG(ERR,
6944                 "port %u get aged flows is not supported.",
6945                  dev->data->port_id);
6946         return -ENOTSUP;
6947 }
6948
6949 /* Wrapper for driver action_validate op callback */
6950 static int
6951 flow_drv_action_validate(struct rte_eth_dev *dev,
6952                          const struct rte_flow_shared_action_conf *conf,
6953                          const struct rte_flow_action *action,
6954                          const struct mlx5_flow_driver_ops *fops,
6955                          struct rte_flow_error *error)
6956 {
6957         static const char err_msg[] = "shared action validation unsupported";
6958
6959         if (!fops->action_validate) {
6960                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6961                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6962                                    NULL, err_msg);
6963                 return -rte_errno;
6964         }
6965         return fops->action_validate(dev, conf, action, error);
6966 }
6967
6968 /**
6969  * Destroys the shared action by handle.
6970  *
6971  * @param dev
6972  *   Pointer to Ethernet device structure.
6973  * @param[in] action
6974  *   Handle for the shared action to be destroyed.
6975  * @param[out] error
6976  *   Perform verbose error reporting if not NULL. PMDs initialize this
6977  *   structure in case of error only.
6978  *
6979  * @return
6980  *   0 on success, a negative errno value otherwise and rte_errno is set.
6981  *
6982  * @note: wrapper for driver action_create op callback.
6983  */
6984 static int
6985 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
6986                            struct rte_flow_shared_action *action,
6987                            struct rte_flow_error *error)
6988 {
6989         static const char err_msg[] = "shared action destruction unsupported";
6990         struct rte_flow_attr attr = { .transfer = 0 };
6991         const struct mlx5_flow_driver_ops *fops =
6992                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
6993
6994         if (!fops->action_destroy) {
6995                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6996                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6997                                    NULL, err_msg);
6998                 return -rte_errno;
6999         }
7000         return fops->action_destroy(dev, action, error);
7001 }
7002
7003 /* Wrapper for driver action_destroy op callback */
7004 static int
7005 flow_drv_action_update(struct rte_eth_dev *dev,
7006                        struct rte_flow_shared_action *action,
7007                        const void *action_conf,
7008                        const struct mlx5_flow_driver_ops *fops,
7009                        struct rte_flow_error *error)
7010 {
7011         static const char err_msg[] = "shared action update unsupported";
7012
7013         if (!fops->action_update) {
7014                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7015                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7016                                    NULL, err_msg);
7017                 return -rte_errno;
7018         }
7019         return fops->action_update(dev, action, action_conf, error);
7020 }
7021
7022 /* Wrapper for driver action_destroy op callback */
7023 static int
7024 flow_drv_action_query(struct rte_eth_dev *dev,
7025                       const struct rte_flow_shared_action *action,
7026                       void *data,
7027                       const struct mlx5_flow_driver_ops *fops,
7028                       struct rte_flow_error *error)
7029 {
7030         static const char err_msg[] = "shared action query unsupported";
7031
7032         if (!fops->action_query) {
7033                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7034                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7035                                    NULL, err_msg);
7036                 return -rte_errno;
7037         }
7038         return fops->action_query(dev, action, data, error);
7039 }
7040
7041 /**
7042  * Create shared action for reuse in multiple flow rules.
7043  *
7044  * @param dev
7045  *   Pointer to Ethernet device structure.
7046  * @param[in] action
7047  *   Action configuration for shared action creation.
7048  * @param[out] error
7049  *   Perform verbose error reporting if not NULL. PMDs initialize this
7050  *   structure in case of error only.
7051  * @return
7052  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7053  */
7054 static struct rte_flow_shared_action *
7055 mlx5_shared_action_create(struct rte_eth_dev *dev,
7056                           const struct rte_flow_shared_action_conf *conf,
7057                           const struct rte_flow_action *action,
7058                           struct rte_flow_error *error)
7059 {
7060         static const char err_msg[] = "shared action creation unsupported";
7061         struct rte_flow_attr attr = { .transfer = 0 };
7062         const struct mlx5_flow_driver_ops *fops =
7063                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7064
7065         if (flow_drv_action_validate(dev, conf, action, fops, error))
7066                 return NULL;
7067         if (!fops->action_create) {
7068                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7069                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7070                                    NULL, err_msg);
7071                 return NULL;
7072         }
7073         return fops->action_create(dev, conf, action, error);
7074 }
7075
7076 /**
7077  * Updates inplace the shared action configuration pointed by *action* handle
7078  * with the configuration provided as *action* argument.
7079  * The update of the shared action configuration effects all flow rules reusing
7080  * the action via handle.
7081  *
7082  * @param dev
7083  *   Pointer to Ethernet device structure.
7084  * @param[in] shared_action
7085  *   Handle for the shared action to be updated.
7086  * @param[in] action
7087  *   Action specification used to modify the action pointed by handle.
7088  *   *action* should be of same type with the action pointed by the *action*
7089  *   handle argument, otherwise considered as invalid.
7090  * @param[out] error
7091  *   Perform verbose error reporting if not NULL. PMDs initialize this
7092  *   structure in case of error only.
7093  *
7094  * @return
7095  *   0 on success, a negative errno value otherwise and rte_errno is set.
7096  */
7097 static int
7098 mlx5_shared_action_update(struct rte_eth_dev *dev,
7099                 struct rte_flow_shared_action *shared_action,
7100                 const struct rte_flow_action *action,
7101                 struct rte_flow_error *error)
7102 {
7103         struct rte_flow_attr attr = { .transfer = 0 };
7104         const struct mlx5_flow_driver_ops *fops =
7105                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7106         int ret;
7107
7108         ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7109         if (ret)
7110                 return ret;
7111         return flow_drv_action_update(dev, shared_action, action->conf, fops,
7112                                       error);
7113 }
7114
7115 /**
7116  * Query the shared action by handle.
7117  *
7118  * This function allows retrieving action-specific data such as counters.
7119  * Data is gathered by special action which may be present/referenced in
7120  * more than one flow rule definition.
7121  *
7122  * \see RTE_FLOW_ACTION_TYPE_COUNT
7123  *
7124  * @param dev
7125  *   Pointer to Ethernet device structure.
7126  * @param[in] action
7127  *   Handle for the shared action to query.
7128  * @param[in, out] data
7129  *   Pointer to storage for the associated query data type.
7130  * @param[out] error
7131  *   Perform verbose error reporting if not NULL. PMDs initialize this
7132  *   structure in case of error only.
7133  *
7134  * @return
7135  *   0 on success, a negative errno value otherwise and rte_errno is set.
7136  */
7137 static int
7138 mlx5_shared_action_query(struct rte_eth_dev *dev,
7139                          const struct rte_flow_shared_action *action,
7140                          void *data,
7141                          struct rte_flow_error *error)
7142 {
7143         struct rte_flow_attr attr = { .transfer = 0 };
7144         const struct mlx5_flow_driver_ops *fops =
7145                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7146
7147         return flow_drv_action_query(dev, action, data, fops, error);
7148 }
7149
7150 /**
7151  * Destroy all shared actions.
7152  *
7153  * @param dev
7154  *   Pointer to Ethernet device.
7155  *
7156  * @return
7157  *   0 on success, a negative errno value otherwise and rte_errno is set.
7158  */
7159 int
7160 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7161 {
7162         struct rte_flow_error error;
7163         struct mlx5_priv *priv = dev->data->dev_private;
7164         struct mlx5_shared_action_rss *action;
7165         int ret = 0;
7166         uint32_t idx;
7167
7168         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7169                       priv->rss_shared_actions, idx, action, next) {
7170                 ret |= mlx5_shared_action_destroy(dev,
7171                        (struct rte_flow_shared_action *)(uintptr_t)idx, &error);
7172         }
7173         return ret;
7174 }
7175
7176 #ifndef HAVE_MLX5DV_DR
7177 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
7178 #else
7179 #define MLX5_DOMAIN_SYNC_FLOW \
7180         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
7181 #endif
7182
7183 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
7184 {
7185         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
7186         const struct mlx5_flow_driver_ops *fops;
7187         int ret;
7188         struct rte_flow_attr attr = { .transfer = 0 };
7189
7190         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7191         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
7192         if (ret > 0)
7193                 ret = -ret;
7194         return ret;
7195 }
7196
7197 /**
7198  * tunnel offload functionalilty is defined for DV environment only
7199  */
7200 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7201 __extension__
7202 union tunnel_offload_mark {
7203         uint32_t val;
7204         struct {
7205                 uint32_t app_reserve:8;
7206                 uint32_t table_id:15;
7207                 uint32_t transfer:1;
7208                 uint32_t _unused_:8;
7209         };
7210 };
7211
7212 static bool
7213 mlx5_access_tunnel_offload_db
7214         (struct rte_eth_dev *dev,
7215          bool (*match)(struct rte_eth_dev *,
7216                        struct mlx5_flow_tunnel *, const void *),
7217          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7218          void (*miss)(struct rte_eth_dev *, void *),
7219          void *ctx, bool lock_op);
7220
7221 static int
7222 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
7223                              struct rte_flow *flow,
7224                              const struct rte_flow_attr *attr,
7225                              const struct rte_flow_action *app_actions,
7226                              uint32_t flow_idx,
7227                              struct tunnel_default_miss_ctx *ctx,
7228                              struct rte_flow_error *error)
7229 {
7230         struct mlx5_priv *priv = dev->data->dev_private;
7231         struct mlx5_flow *dev_flow;
7232         struct rte_flow_attr miss_attr = *attr;
7233         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
7234         const struct rte_flow_item miss_items[2] = {
7235                 {
7236                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7237                         .spec = NULL,
7238                         .last = NULL,
7239                         .mask = NULL
7240                 },
7241                 {
7242                         .type = RTE_FLOW_ITEM_TYPE_END,
7243                         .spec = NULL,
7244                         .last = NULL,
7245                         .mask = NULL
7246                 }
7247         };
7248         union tunnel_offload_mark mark_id;
7249         struct rte_flow_action_mark miss_mark;
7250         struct rte_flow_action miss_actions[3] = {
7251                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
7252                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
7253         };
7254         const struct rte_flow_action_jump *jump_data;
7255         uint32_t i, flow_table = 0; /* prevent compilation warning */
7256         struct flow_grp_info grp_info = {
7257                 .external = 1,
7258                 .transfer = attr->transfer,
7259                 .fdb_def_rule = !!priv->fdb_def_rule,
7260                 .std_tbl_fix = 0,
7261         };
7262         int ret;
7263
7264         if (!attr->transfer) {
7265                 uint32_t q_size;
7266
7267                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
7268                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
7269                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
7270                                          0, SOCKET_ID_ANY);
7271                 if (!ctx->queue)
7272                         return rte_flow_error_set
7273                                 (error, ENOMEM,
7274                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7275                                 NULL, "invalid default miss RSS");
7276                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7277                 ctx->action_rss.level = 0,
7278                 ctx->action_rss.types = priv->rss_conf.rss_hf,
7279                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
7280                 ctx->action_rss.queue_num = priv->reta_idx_n,
7281                 ctx->action_rss.key = priv->rss_conf.rss_key,
7282                 ctx->action_rss.queue = ctx->queue;
7283                 if (!priv->reta_idx_n || !priv->rxqs_n)
7284                         return rte_flow_error_set
7285                                 (error, EINVAL,
7286                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7287                                 NULL, "invalid port configuration");
7288                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7289                         ctx->action_rss.types = 0;
7290                 for (i = 0; i != priv->reta_idx_n; ++i)
7291                         ctx->queue[i] = (*priv->reta_idx)[i];
7292         } else {
7293                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
7294                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
7295         }
7296         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
7297         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
7298         jump_data = app_actions->conf;
7299         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
7300         miss_attr.group = jump_data->group;
7301         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
7302                                        &flow_table, &grp_info, error);
7303         if (ret)
7304                 return rte_flow_error_set(error, EINVAL,
7305                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7306                                           NULL, "invalid tunnel id");
7307         mark_id.app_reserve = 0;
7308         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
7309         mark_id.transfer = !!attr->transfer;
7310         mark_id._unused_ = 0;
7311         miss_mark.id = mark_id.val;
7312         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
7313                                     miss_items, miss_actions, flow_idx, error);
7314         if (!dev_flow)
7315                 return -rte_errno;
7316         dev_flow->flow = flow;
7317         dev_flow->external = true;
7318         dev_flow->tunnel = tunnel;
7319         /* Subflow object was created, we must include one in the list. */
7320         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7321                       dev_flow->handle, next);
7322         DRV_LOG(DEBUG,
7323                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
7324                 dev->data->port_id, tunnel->app_tunnel.type,
7325                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
7326         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
7327                                   miss_actions, error);
7328         if (!ret)
7329                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
7330                                                   error);
7331
7332         return ret;
7333 }
7334
7335 static const struct mlx5_flow_tbl_data_entry  *
7336 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7337 {
7338         struct mlx5_priv *priv = dev->data->dev_private;
7339         struct mlx5_dev_ctx_shared *sh = priv->sh;
7340         struct mlx5_hlist_entry *he;
7341         union tunnel_offload_mark mbits = { .val = mark };
7342         union mlx5_flow_tbl_key table_key = {
7343                 {
7344                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7345                         .dummy = 0,
7346                         .domain = !!mbits.transfer,
7347                         .direction = 0,
7348                 }
7349         };
7350         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
7351         return he ?
7352                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7353 }
7354
7355 static void
7356 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
7357                                    struct mlx5_hlist_entry *entry)
7358 {
7359         struct mlx5_dev_ctx_shared *sh = list->ctx;
7360         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7361
7362         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7363                         tunnel_flow_tbl_to_id(tte->flow_table));
7364         mlx5_free(tte);
7365 }
7366
7367 static int
7368 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
7369                                   struct mlx5_hlist_entry *entry,
7370                                   uint64_t key, void *cb_ctx __rte_unused)
7371 {
7372         union tunnel_tbl_key tbl = {
7373                 .val = key,
7374         };
7375         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7376
7377         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
7378 }
7379
7380 static struct mlx5_hlist_entry *
7381 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
7382                                    void *ctx __rte_unused)
7383 {
7384         struct mlx5_dev_ctx_shared *sh = list->ctx;
7385         struct tunnel_tbl_entry *tte;
7386         union tunnel_tbl_key tbl = {
7387                 .val = key,
7388         };
7389
7390         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7391                           sizeof(*tte), 0,
7392                           SOCKET_ID_ANY);
7393         if (!tte)
7394                 goto err;
7395         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7396                           &tte->flow_table);
7397         if (tte->flow_table >= MLX5_MAX_TABLES) {
7398                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
7399                         tte->flow_table);
7400                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7401                                 tte->flow_table);
7402                 goto err;
7403         } else if (!tte->flow_table) {
7404                 goto err;
7405         }
7406         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7407         tte->tunnel_id = tbl.tunnel_id;
7408         tte->group = tbl.group;
7409         return &tte->hash;
7410 err:
7411         if (tte)
7412                 mlx5_free(tte);
7413         return NULL;
7414 }
7415
7416 static uint32_t
7417 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7418                                 const struct mlx5_flow_tunnel *tunnel,
7419                                 uint32_t group, uint32_t *table,
7420                                 struct rte_flow_error *error)
7421 {
7422         struct mlx5_hlist_entry *he;
7423         struct tunnel_tbl_entry *tte;
7424         union tunnel_tbl_key key = {
7425                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7426                 .group = group
7427         };
7428         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7429         struct mlx5_hlist *group_hash;
7430
7431         group_hash = tunnel ? tunnel->groups : thub->groups;
7432         he = mlx5_hlist_register(group_hash, key.val, NULL);
7433         if (!he)
7434                 return rte_flow_error_set(error, EINVAL,
7435                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7436                                           NULL,
7437                                           "tunnel group index not supported");
7438         tte = container_of(he, typeof(*tte), hash);
7439         *table = tte->flow_table;
7440         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7441                 dev->data->port_id, key.tunnel_id, group, *table);
7442         return 0;
7443 }
7444
7445 static void
7446 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
7447                       struct mlx5_flow_tunnel *tunnel)
7448 {
7449         struct mlx5_priv *priv = dev->data->dev_private;
7450         struct mlx5_indexed_pool *ipool;
7451
7452         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
7453                 dev->data->port_id, tunnel->tunnel_id);
7454         LIST_REMOVE(tunnel, chain);
7455         mlx5_hlist_destroy(tunnel->groups);
7456         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7457         mlx5_ipool_free(ipool, tunnel->tunnel_id);
7458 }
7459
7460 static bool
7461 mlx5_access_tunnel_offload_db
7462         (struct rte_eth_dev *dev,
7463          bool (*match)(struct rte_eth_dev *,
7464                        struct mlx5_flow_tunnel *, const void *),
7465          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7466          void (*miss)(struct rte_eth_dev *, void *),
7467          void *ctx, bool lock_op)
7468 {
7469         bool verdict = false;
7470         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7471         struct mlx5_flow_tunnel *tunnel;
7472
7473         rte_spinlock_lock(&thub->sl);
7474         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
7475                 verdict = match(dev, tunnel, (const void *)ctx);
7476                 if (verdict)
7477                         break;
7478         }
7479         if (!lock_op)
7480                 rte_spinlock_unlock(&thub->sl);
7481         if (verdict && hit)
7482                 hit(dev, tunnel, ctx);
7483         if (!verdict && miss)
7484                 miss(dev, ctx);
7485         if (lock_op)
7486                 rte_spinlock_unlock(&thub->sl);
7487
7488         return verdict;
7489 }
7490
7491 struct tunnel_db_find_tunnel_id_ctx {
7492         uint32_t tunnel_id;
7493         struct mlx5_flow_tunnel *tunnel;
7494 };
7495
7496 static bool
7497 find_tunnel_id_match(struct rte_eth_dev *dev,
7498                      struct mlx5_flow_tunnel *tunnel, const void *x)
7499 {
7500         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7501
7502         RTE_SET_USED(dev);
7503         return tunnel->tunnel_id == ctx->tunnel_id;
7504 }
7505
7506 static void
7507 find_tunnel_id_hit(struct rte_eth_dev *dev,
7508                    struct mlx5_flow_tunnel *tunnel, void *x)
7509 {
7510         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7511         RTE_SET_USED(dev);
7512         ctx->tunnel = tunnel;
7513 }
7514
7515 static struct mlx5_flow_tunnel *
7516 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
7517 {
7518         struct tunnel_db_find_tunnel_id_ctx ctx = {
7519                 .tunnel_id = id,
7520         };
7521
7522         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
7523                                       find_tunnel_id_hit, NULL, &ctx, true);
7524
7525         return ctx.tunnel;
7526 }
7527
7528 static struct mlx5_flow_tunnel *
7529 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
7530                           const struct rte_flow_tunnel *app_tunnel)
7531 {
7532         struct mlx5_priv *priv = dev->data->dev_private;
7533         struct mlx5_indexed_pool *ipool;
7534         struct mlx5_flow_tunnel *tunnel;
7535         uint32_t id;
7536
7537         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7538         tunnel = mlx5_ipool_zmalloc(ipool, &id);
7539         if (!tunnel)
7540                 return NULL;
7541         if (id >= MLX5_MAX_TUNNELS) {
7542                 mlx5_ipool_free(ipool, id);
7543                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
7544                 return NULL;
7545         }
7546         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
7547                                            mlx5_flow_tunnel_grp2tbl_create_cb,
7548                                            mlx5_flow_tunnel_grp2tbl_match_cb,
7549                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
7550         if (!tunnel->groups) {
7551                 mlx5_ipool_free(ipool, id);
7552                 return NULL;
7553         }
7554         tunnel->groups->ctx = priv->sh;
7555         /* initiate new PMD tunnel */
7556         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
7557         tunnel->tunnel_id = id;
7558         tunnel->action.type = (typeof(tunnel->action.type))
7559                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
7560         tunnel->action.conf = tunnel;
7561         tunnel->item.type = (typeof(tunnel->item.type))
7562                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
7563         tunnel->item.spec = tunnel;
7564         tunnel->item.last = NULL;
7565         tunnel->item.mask = NULL;
7566
7567         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
7568                 dev->data->port_id, tunnel->tunnel_id);
7569
7570         return tunnel;
7571 }
7572
7573 struct tunnel_db_get_tunnel_ctx {
7574         const struct rte_flow_tunnel *app_tunnel;
7575         struct mlx5_flow_tunnel *tunnel;
7576 };
7577
7578 static bool get_tunnel_match(struct rte_eth_dev *dev,
7579                              struct mlx5_flow_tunnel *tunnel, const void *x)
7580 {
7581         const struct tunnel_db_get_tunnel_ctx *ctx = x;
7582
7583         RTE_SET_USED(dev);
7584         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
7585                        sizeof(*ctx->app_tunnel));
7586 }
7587
7588 static void get_tunnel_hit(struct rte_eth_dev *dev,
7589                            struct mlx5_flow_tunnel *tunnel, void *x)
7590 {
7591         /* called under tunnel spinlock protection */
7592         struct tunnel_db_get_tunnel_ctx *ctx = x;
7593
7594         RTE_SET_USED(dev);
7595         tunnel->refctn++;
7596         ctx->tunnel = tunnel;
7597 }
7598
7599 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
7600 {
7601         /* called under tunnel spinlock protection */
7602         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7603         struct tunnel_db_get_tunnel_ctx *ctx = x;
7604
7605         rte_spinlock_unlock(&thub->sl);
7606         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
7607         ctx->tunnel->refctn = 1;
7608         rte_spinlock_lock(&thub->sl);
7609         if (ctx->tunnel)
7610                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
7611 }
7612
7613
7614 static int
7615 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
7616                      const struct rte_flow_tunnel *app_tunnel,
7617                      struct mlx5_flow_tunnel **tunnel)
7618 {
7619         struct tunnel_db_get_tunnel_ctx ctx = {
7620                 .app_tunnel = app_tunnel,
7621         };
7622
7623         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
7624                                       get_tunnel_miss, &ctx, true);
7625         *tunnel = ctx.tunnel;
7626         return ctx.tunnel ? 0 : -ENOMEM;
7627 }
7628
7629 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
7630 {
7631         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
7632
7633         if (!thub)
7634                 return;
7635         if (!LIST_EMPTY(&thub->tunnels))
7636                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
7637         mlx5_hlist_destroy(thub->groups);
7638         mlx5_free(thub);
7639 }
7640
7641 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
7642 {
7643         int err;
7644         struct mlx5_flow_tunnel_hub *thub;
7645
7646         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
7647                            0, SOCKET_ID_ANY);
7648         if (!thub)
7649                 return -ENOMEM;
7650         LIST_INIT(&thub->tunnels);
7651         rte_spinlock_init(&thub->sl);
7652         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
7653                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
7654                                          mlx5_flow_tunnel_grp2tbl_match_cb,
7655                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
7656         if (!thub->groups) {
7657                 err = -rte_errno;
7658                 goto err;
7659         }
7660         thub->groups->ctx = sh;
7661         sh->tunnel_hub = thub;
7662
7663         return 0;
7664
7665 err:
7666         if (thub->groups)
7667                 mlx5_hlist_destroy(thub->groups);
7668         if (thub)
7669                 mlx5_free(thub);
7670         return err;
7671 }
7672
7673 static inline bool
7674 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
7675                           struct rte_flow_tunnel *tunnel,
7676                           const char *err_msg)
7677 {
7678         err_msg = NULL;
7679         if (!is_tunnel_offload_active(dev)) {
7680                 err_msg = "tunnel offload was not activated";
7681                 goto out;
7682         } else if (!tunnel) {
7683                 err_msg = "no application tunnel";
7684                 goto out;
7685         }
7686
7687         switch (tunnel->type) {
7688         default:
7689                 err_msg = "unsupported tunnel type";
7690                 goto out;
7691         case RTE_FLOW_ITEM_TYPE_VXLAN:
7692                 break;
7693         }
7694
7695 out:
7696         return !err_msg;
7697 }
7698
7699 static int
7700 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
7701                     struct rte_flow_tunnel *app_tunnel,
7702                     struct rte_flow_action **actions,
7703                     uint32_t *num_of_actions,
7704                     struct rte_flow_error *error)
7705 {
7706         int ret;
7707         struct mlx5_flow_tunnel *tunnel;
7708         const char *err_msg = NULL;
7709         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7710
7711         if (!verdict)
7712                 return rte_flow_error_set(error, EINVAL,
7713                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7714                                           err_msg);
7715         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7716         if (ret < 0) {
7717                 return rte_flow_error_set(error, ret,
7718                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7719                                           "failed to initialize pmd tunnel");
7720         }
7721         *actions = &tunnel->action;
7722         *num_of_actions = 1;
7723         return 0;
7724 }
7725
7726 static int
7727 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
7728                        struct rte_flow_tunnel *app_tunnel,
7729                        struct rte_flow_item **items,
7730                        uint32_t *num_of_items,
7731                        struct rte_flow_error *error)
7732 {
7733         int ret;
7734         struct mlx5_flow_tunnel *tunnel;
7735         const char *err_msg = NULL;
7736         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7737
7738         if (!verdict)
7739                 return rte_flow_error_set(error, EINVAL,
7740                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7741                                           err_msg);
7742         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7743         if (ret < 0) {
7744                 return rte_flow_error_set(error, ret,
7745                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7746                                           "failed to initialize pmd tunnel");
7747         }
7748         *items = &tunnel->item;
7749         *num_of_items = 1;
7750         return 0;
7751 }
7752
7753 struct tunnel_db_element_release_ctx {
7754         struct rte_flow_item *items;
7755         struct rte_flow_action *actions;
7756         uint32_t num_elements;
7757         struct rte_flow_error *error;
7758         int ret;
7759 };
7760
7761 static bool
7762 tunnel_element_release_match(struct rte_eth_dev *dev,
7763                              struct mlx5_flow_tunnel *tunnel, const void *x)
7764 {
7765         const struct tunnel_db_element_release_ctx *ctx = x;
7766
7767         RTE_SET_USED(dev);
7768         if (ctx->num_elements != 1)
7769                 return false;
7770         else if (ctx->items)
7771                 return ctx->items == &tunnel->item;
7772         else if (ctx->actions)
7773                 return ctx->actions == &tunnel->action;
7774
7775         return false;
7776 }
7777
7778 static void
7779 tunnel_element_release_hit(struct rte_eth_dev *dev,
7780                            struct mlx5_flow_tunnel *tunnel, void *x)
7781 {
7782         struct tunnel_db_element_release_ctx *ctx = x;
7783         ctx->ret = 0;
7784         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7785                 mlx5_flow_tunnel_free(dev, tunnel);
7786 }
7787
7788 static void
7789 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
7790 {
7791         struct tunnel_db_element_release_ctx *ctx = x;
7792         RTE_SET_USED(dev);
7793         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
7794                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7795                                       "invalid argument");
7796 }
7797
7798 static int
7799 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
7800                        struct rte_flow_item *pmd_items,
7801                        uint32_t num_items, struct rte_flow_error *err)
7802 {
7803         struct tunnel_db_element_release_ctx ctx = {
7804                 .items = pmd_items,
7805                 .actions = NULL,
7806                 .num_elements = num_items,
7807                 .error = err,
7808         };
7809
7810         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7811                                       tunnel_element_release_hit,
7812                                       tunnel_element_release_miss, &ctx, false);
7813
7814         return ctx.ret;
7815 }
7816
7817 static int
7818 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
7819                          struct rte_flow_action *pmd_actions,
7820                          uint32_t num_actions, struct rte_flow_error *err)
7821 {
7822         struct tunnel_db_element_release_ctx ctx = {
7823                 .items = NULL,
7824                 .actions = pmd_actions,
7825                 .num_elements = num_actions,
7826                 .error = err,
7827         };
7828
7829         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7830                                       tunnel_element_release_hit,
7831                                       tunnel_element_release_miss, &ctx, false);
7832
7833         return ctx.ret;
7834 }
7835
7836 static int
7837 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
7838                                   struct rte_mbuf *m,
7839                                   struct rte_flow_restore_info *info,
7840                                   struct rte_flow_error *err)
7841 {
7842         uint64_t ol_flags = m->ol_flags;
7843         const struct mlx5_flow_tbl_data_entry *tble;
7844         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
7845
7846         if (!is_tunnel_offload_active(dev)) {
7847                 info->flags = 0;
7848                 return 0;
7849         }
7850
7851         if ((ol_flags & mask) != mask)
7852                 goto err;
7853         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
7854         if (!tble) {
7855                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
7856                         dev->data->port_id, m->hash.fdir.hi);
7857                 goto err;
7858         }
7859         MLX5_ASSERT(tble->tunnel);
7860         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
7861         info->group_id = tble->group_id;
7862         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
7863                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
7864                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
7865
7866         return 0;
7867
7868 err:
7869         return rte_flow_error_set(err, EINVAL,
7870                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7871                                   "failed to get restore info");
7872 }
7873
7874 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
7875 static int
7876 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
7877                            __rte_unused struct rte_flow_tunnel *app_tunnel,
7878                            __rte_unused struct rte_flow_action **actions,
7879                            __rte_unused uint32_t *num_of_actions,
7880                            __rte_unused struct rte_flow_error *error)
7881 {
7882         return -ENOTSUP;
7883 }
7884
7885 static int
7886 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
7887                        __rte_unused struct rte_flow_tunnel *app_tunnel,
7888                        __rte_unused struct rte_flow_item **items,
7889                        __rte_unused uint32_t *num_of_items,
7890                        __rte_unused struct rte_flow_error *error)
7891 {
7892         return -ENOTSUP;
7893 }
7894
7895 static int
7896 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
7897                               __rte_unused struct rte_flow_item *pmd_items,
7898                               __rte_unused uint32_t num_items,
7899                               __rte_unused struct rte_flow_error *err)
7900 {
7901         return -ENOTSUP;
7902 }
7903
7904 static int
7905 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
7906                                 __rte_unused struct rte_flow_action *pmd_action,
7907                                 __rte_unused uint32_t num_actions,
7908                                 __rte_unused struct rte_flow_error *err)
7909 {
7910         return -ENOTSUP;
7911 }
7912
7913 static int
7914 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
7915                                   __rte_unused struct rte_mbuf *m,
7916                                   __rte_unused struct rte_flow_restore_info *i,
7917                                   __rte_unused struct rte_flow_error *err)
7918 {
7919         return -ENOTSUP;
7920 }
7921
7922 static int
7923 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
7924                              __rte_unused struct rte_flow *flow,
7925                              __rte_unused const struct rte_flow_attr *attr,
7926                              __rte_unused const struct rte_flow_action *actions,
7927                              __rte_unused uint32_t flow_idx,
7928                              __rte_unused struct tunnel_default_miss_ctx *ctx,
7929                              __rte_unused struct rte_flow_error *error)
7930 {
7931         return -ENOTSUP;
7932 }
7933
7934 static struct mlx5_flow_tunnel *
7935 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
7936                     __rte_unused uint32_t id)
7937 {
7938         return NULL;
7939 }
7940
7941 static void
7942 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
7943                       __rte_unused struct mlx5_flow_tunnel *tunnel)
7944 {
7945 }
7946
7947 static uint32_t
7948 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
7949                                 __rte_unused const struct mlx5_flow_tunnel *t,
7950                                 __rte_unused uint32_t group,
7951                                 __rte_unused uint32_t *table,
7952                                 struct rte_flow_error *error)
7953 {
7954         return rte_flow_error_set(error, ENOTSUP,
7955                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7956                                   "tunnel offload requires DV support");
7957 }
7958
7959 void
7960 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
7961                         __rte_unused  uint16_t port_id)
7962 {
7963 }
7964 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
7965