drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              struct tunnel_default_miss_ctx *ctx,
  54                              struct rte_flow_error *error);
  55 static struct mlx5_flow_tunnel *
  56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  57 static void
  58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  59 static uint32_t
  60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  61                                 const struct mlx5_flow_tunnel *tunnel,
  62                                 uint32_t group, uint32_t *table,
  63                                 struct rte_flow_error *error);
  64
  65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  66 static void mlx5_flow_pop_thread_workspace(void);
  67
  68
  69 /** Device flow drivers. */
  70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  71
  72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  73
  74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  75         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  77         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  78 #endif
  79         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  80         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  81 };
  82
  83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  85         (const int []){ \
  86                 __VA_ARGS__, 0, \
  87         }
  88
  89 /** Node object of input graph for mlx5_flow_expand_rss(). */
  90 struct mlx5_flow_expand_node {
  91         const int *const next;
  92         /**<
  93          * List of next node indexes. Index 0 is interpreted as a terminator.
  94          */
  95         const enum rte_flow_item_type type;
  96         /**< Pattern item type of current node. */
  97         uint64_t rss_types;
  98         /**<
  99          * RSS types bit-field associated with this node
 100          * (see ETH_RSS_* definitions).
 101          */
 102 };
 103
 104 /** Object returned by mlx5_flow_expand_rss(). */
 105 struct mlx5_flow_expand_rss {
 106         uint32_t entries;
 107         /**< Number of entries @p patterns and @p priorities. */
 108         struct {
 109                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 110                 uint32_t priority; /**< Priority offset for each expansion. */
 111         } entry[];
 112 };
 113
 114 static enum rte_flow_item_type
 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 116 {
 117         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 118         uint16_t ether_type = 0;
 119         uint16_t ether_type_m;
 120         uint8_t ip_next_proto = 0;
 121         uint8_t ip_next_proto_m;
 122
 123         if (item == NULL || item->spec == NULL)
 124                 return ret;
 125         switch (item->type) {
 126         case RTE_FLOW_ITEM_TYPE_ETH:
 127                 if (item->mask)
 128                         ether_type_m = ((const struct rte_flow_item_eth *)
 129                                                 (item->mask))->type;
 130                 else
 131                         ether_type_m = rte_flow_item_eth_mask.type;
 132                 if (ether_type_m != RTE_BE16(0xFFFF))
 133                         break;
 134                 ether_type = ((const struct rte_flow_item_eth *)
 135                                 (item->spec))->type;
 136                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 139                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 140                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 141                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 142                 else
 143                         ret = RTE_FLOW_ITEM_TYPE_END;
 144                 break;
 145         case RTE_FLOW_ITEM_TYPE_VLAN:
 146                 if (item->mask)
 147                         ether_type_m = ((const struct rte_flow_item_vlan *)
 148                                                 (item->mask))->inner_type;
 149                 else
 150                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 151                 if (ether_type_m != RTE_BE16(0xFFFF))
 152                         break;
 153                 ether_type = ((const struct rte_flow_item_vlan *)
 154                                 (item->spec))->inner_type;
 155                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 156                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 157                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 159                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 160                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV4:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 167                                         (item->mask))->hdr.next_proto_id;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 174                                 (item->spec))->hdr.next_proto_id;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         case RTE_FLOW_ITEM_TYPE_IPV6:
 187                 if (item->mask)
 188                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 189                                                 (item->mask))->hdr.proto;
 190                 else
 191                         ip_next_proto_m =
 192                                 rte_flow_item_ipv6_mask.hdr.proto;
 193                 if (ip_next_proto_m != 0xFF)
 194                         break;
 195                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 196                                 (item->spec))->hdr.proto;
 197                 if (ip_next_proto == IPPROTO_UDP)
 198                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 199                 else if (ip_next_proto == IPPROTO_TCP)
 200                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 201                 else if (ip_next_proto == IPPROTO_IP)
 202                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 203                 else if (ip_next_proto == IPPROTO_IPV6)
 204                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 205                 else
 206                         ret = RTE_FLOW_ITEM_TYPE_END;
 207                 break;
 208         default:
 209                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 210                 break;
 211         }
 212         return ret;
 213 }
 214
 215 /**
 216  * Expand RSS flows into several possible flows according to the RSS hash
 217  * fields requested and the driver capabilities.
 218  *
 219  * @param[out] buf
 220  *   Buffer to store the result expansion.
 221  * @param[in] size
 222  *   Buffer size in bytes. If 0, @p buf can be NULL.
 223  * @param[in] pattern
 224  *   User flow pattern.
 225  * @param[in] types
 226  *   RSS types to expand (see ETH_RSS_* definitions).
 227  * @param[in] graph
 228  *   Input graph to expand @p pattern according to @p types.
 229  * @param[in] graph_root_index
 230  *   Index of root node in @p graph, typically 0.
 231  *
 232  * @return
 233  *   A positive value representing the size of @p buf in bytes regardless of
 234  *   @p size on success, a negative errno value otherwise and rte_errno is
 235  *   set, the following errors are defined:
 236  *
 237  *   -E2BIG: graph-depth @p graph is too deep.
 238  */
 239 static int
 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 241                      const struct rte_flow_item *pattern, uint64_t types,
 242                      const struct mlx5_flow_expand_node graph[],
 243                      int graph_root_index)
 244 {
 245         const int elt_n = 8;
 246         const struct rte_flow_item *item;
 247         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 248         const int *next_node;
 249         const int *stack[elt_n];
 250         int stack_pos = 0;
 251         struct rte_flow_item flow_items[elt_n];
 252         unsigned int i;
 253         size_t lsize;
 254         size_t user_pattern_size = 0;
 255         void *addr = NULL;
 256         const struct mlx5_flow_expand_node *next = NULL;
 257         struct rte_flow_item missed_item;
 258         int missed = 0;
 259         int elt = 0;
 260         const struct rte_flow_item *last_item = NULL;
 261
 262         memset(&missed_item, 0, sizeof(missed_item));
 263         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 264                 elt_n * sizeof(buf->entry[0]);
 265         if (lsize <= size) {
 266                 buf->entry[0].priority = 0;
 267                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 268                 buf->entries = 0;
 269                 addr = buf->entry[0].pattern;
 270         }
 271         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 272                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 273                         last_item = item;
 274                 for (i = 0; node->next && node->next[i]; ++i) {
 275                         next = &graph[node->next[i]];
 276                         if (next->type == item->type)
 277                                 break;
 278                 }
 279                 if (next)
 280                         node = next;
 281                 user_pattern_size += sizeof(*item);
 282         }
 283         user_pattern_size += sizeof(*item); /* Handle END item. */
 284         lsize += user_pattern_size;
 285         /* Copy the user pattern in the first entry of the buffer. */
 286         if (lsize <= size) {
 287                 rte_memcpy(addr, pattern, user_pattern_size);
 288                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 289                 buf->entries = 1;
 290         }
 291         /* Start expanding. */
 292         memset(flow_items, 0, sizeof(flow_items));
 293         user_pattern_size -= sizeof(*item);
 294         /*
 295          * Check if the last valid item has spec set, need complete pattern,
 296          * and the pattern can be used for expansion.
 297          */
 298         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 299         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 300                 /* Item type END indicates expansion is not required. */
 301                 return lsize;
 302         }
 303         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 304                 next = NULL;
 305                 missed = 1;
 306                 for (i = 0; node->next && node->next[i]; ++i) {
 307                         next = &graph[node->next[i]];
 308                         if (next->type == missed_item.type) {
 309                                 flow_items[0].type = missed_item.type;
 310                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 311                                 break;
 312                         }
 313                         next = NULL;
 314                 }
 315         }
 316         if (next && missed) {
 317                 elt = 2; /* missed item + item end. */
 318                 node = next;
 319                 lsize += elt * sizeof(*item) + user_pattern_size;
 320                 if ((node->rss_types & types) && lsize <= size) {
 321                         buf->entry[buf->entries].priority = 1;
 322                         buf->entry[buf->entries].pattern = addr;
 323                         buf->entries++;
 324                         rte_memcpy(addr, buf->entry[0].pattern,
 325                                    user_pattern_size);
 326                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 327                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 328                         addr = (void *)(((uintptr_t)addr) +
 329                                         elt * sizeof(*item));
 330                 }
 331         }
 332         memset(flow_items, 0, sizeof(flow_items));
 333         next_node = node->next;
 334         stack[stack_pos] = next_node;
 335         node = next_node ? &graph[*next_node] : NULL;
 336         while (node) {
 337                 flow_items[stack_pos].type = node->type;
 338                 if (node->rss_types & types) {
 339                         /*
 340                          * compute the number of items to copy from the
 341                          * expansion and copy it.
 342                          * When the stack_pos is 0, there are 1 element in it,
 343                          * plus the addition END item.
 344                          */
 345                         elt = stack_pos + 2;
 346                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 347                         lsize += elt * sizeof(*item) + user_pattern_size;
 348                         if (lsize <= size) {
 349                                 size_t n = elt * sizeof(*item);
 350
 351                                 buf->entry[buf->entries].priority =
 352                                         stack_pos + 1 + missed;
 353                                 buf->entry[buf->entries].pattern = addr;
 354                                 buf->entries++;
 355                                 rte_memcpy(addr, buf->entry[0].pattern,
 356                                            user_pattern_size);
 357                                 addr = (void *)(((uintptr_t)addr) +
 358                                                 user_pattern_size);
 359                                 rte_memcpy(addr, &missed_item,
 360                                            missed * sizeof(*item));
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                         missed * sizeof(*item));
 363                                 rte_memcpy(addr, flow_items, n);
 364                                 addr = (void *)(((uintptr_t)addr) + n);
 365                         }
 366                 }
 367                 /* Go deeper. */
 368                 if (node->next) {
 369                         next_node = node->next;
 370                         if (stack_pos++ == elt_n) {
 371                                 rte_errno = E2BIG;
 372                                 return -rte_errno;
 373                         }
 374                         stack[stack_pos] = next_node;
 375                 } else if (*(next_node + 1)) {
 376                         /* Follow up with the next possibility. */
 377                         ++next_node;
 378                 } else {
 379                         /* Move to the next path. */
 380                         if (stack_pos)
 381                                 next_node = stack[--stack_pos];
 382                         next_node++;
 383                         stack[stack_pos] = next_node;
 384                 }
 385                 node = *next_node ? &graph[*next_node] : NULL;
 386         };
 387         /* no expanded flows but we have missed item, create one rule for it */
 388         if (buf->entries == 1 && missed != 0) {
 389                 elt = 2;
 390                 lsize += elt * sizeof(*item) + user_pattern_size;
 391                 if (lsize <= size) {
 392                         buf->entry[buf->entries].priority = 1;
 393                         buf->entry[buf->entries].pattern = addr;
 394                         buf->entries++;
 395                         flow_items[0].type = missed_item.type;
 396                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 397                         rte_memcpy(addr, buf->entry[0].pattern,
 398                                    user_pattern_size);
 399                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 400                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 401                 }
 402         }
 403         return lsize;
 404 }
 405
 406 enum mlx5_expansion {
 407         MLX5_EXPANSION_ROOT,
 408         MLX5_EXPANSION_ROOT_OUTER,
 409         MLX5_EXPANSION_ROOT_ETH_VLAN,
 410         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 411         MLX5_EXPANSION_OUTER_ETH,
 412         MLX5_EXPANSION_OUTER_ETH_VLAN,
 413         MLX5_EXPANSION_OUTER_VLAN,
 414         MLX5_EXPANSION_OUTER_IPV4,
 415         MLX5_EXPANSION_OUTER_IPV4_UDP,
 416         MLX5_EXPANSION_OUTER_IPV4_TCP,
 417         MLX5_EXPANSION_OUTER_IPV6,
 418         MLX5_EXPANSION_OUTER_IPV6_UDP,
 419         MLX5_EXPANSION_OUTER_IPV6_TCP,
 420         MLX5_EXPANSION_VXLAN,
 421         MLX5_EXPANSION_VXLAN_GPE,
 422         MLX5_EXPANSION_GRE,
 423         MLX5_EXPANSION_MPLS,
 424         MLX5_EXPANSION_ETH,
 425         MLX5_EXPANSION_ETH_VLAN,
 426         MLX5_EXPANSION_VLAN,
 427         MLX5_EXPANSION_IPV4,
 428         MLX5_EXPANSION_IPV4_UDP,
 429         MLX5_EXPANSION_IPV4_TCP,
 430         MLX5_EXPANSION_IPV6,
 431         MLX5_EXPANSION_IPV6_UDP,
 432         MLX5_EXPANSION_IPV6_TCP,
 433 };
 434
 435 /** Supported expansion of items. */
 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 437         [MLX5_EXPANSION_ROOT] = {
 438                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 439                                                   MLX5_EXPANSION_IPV4,
 440                                                   MLX5_EXPANSION_IPV6),
 441                 .type = RTE_FLOW_ITEM_TYPE_END,
 442         },
 443         [MLX5_EXPANSION_ROOT_OUTER] = {
 444                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 445                                                   MLX5_EXPANSION_OUTER_IPV4,
 446                                                   MLX5_EXPANSION_OUTER_IPV6),
 447                 .type = RTE_FLOW_ITEM_TYPE_END,
 448         },
 449         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 450                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 451                 .type = RTE_FLOW_ITEM_TYPE_END,
 452         },
 453         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 454                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 455                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_END,
 457         },
 458         [MLX5_EXPANSION_OUTER_ETH] = {
 459                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 460                                                   MLX5_EXPANSION_OUTER_IPV6,
 461                                                   MLX5_EXPANSION_MPLS),
 462                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 463                 .rss_types = 0,
 464         },
 465         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 466                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 467                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 468                 .rss_types = 0,
 469         },
 470         [MLX5_EXPANSION_OUTER_VLAN] = {
 471                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 472                                                   MLX5_EXPANSION_OUTER_IPV6),
 473                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 477                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 478                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 479                          MLX5_EXPANSION_GRE,
 480                          MLX5_EXPANSION_IPV4,
 481                          MLX5_EXPANSION_IPV6),
 482                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 483                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 484                         ETH_RSS_NONFRAG_IPV4_OTHER,
 485         },
 486         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 487                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 488                                                   MLX5_EXPANSION_VXLAN_GPE),
 489                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 490                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 493                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 494                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 498                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 499                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 500                          MLX5_EXPANSION_IPV4,
 501                          MLX5_EXPANSION_IPV6),
 502                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 503                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 504                         ETH_RSS_NONFRAG_IPV6_OTHER,
 505         },
 506         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 508                                                   MLX5_EXPANSION_VXLAN_GPE),
 509                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 510                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 511         },
 512         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 513                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 514                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 515         },
 516         [MLX5_EXPANSION_VXLAN] = {
 517                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 518                                                   MLX5_EXPANSION_IPV4,
 519                                                   MLX5_EXPANSION_IPV6),
 520                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 521         },
 522         [MLX5_EXPANSION_VXLAN_GPE] = {
 523                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 524                                                   MLX5_EXPANSION_IPV4,
 525                                                   MLX5_EXPANSION_IPV6),
 526                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 527         },
 528         [MLX5_EXPANSION_GRE] = {
 529                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 530                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 531         },
 532         [MLX5_EXPANSION_MPLS] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 534                                                   MLX5_EXPANSION_IPV6),
 535                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 536         },
 537         [MLX5_EXPANSION_ETH] = {
 538                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 539                                                   MLX5_EXPANSION_IPV6),
 540                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 541         },
 542         [MLX5_EXPANSION_ETH_VLAN] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 544                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 545         },
 546         [MLX5_EXPANSION_VLAN] = {
 547                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 548                                                   MLX5_EXPANSION_IPV6),
 549                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 550         },
 551         [MLX5_EXPANSION_IPV4] = {
 552                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 553                                                   MLX5_EXPANSION_IPV4_TCP),
 554                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 555                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 556                         ETH_RSS_NONFRAG_IPV4_OTHER,
 557         },
 558         [MLX5_EXPANSION_IPV4_UDP] = {
 559                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 560                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 561         },
 562         [MLX5_EXPANSION_IPV4_TCP] = {
 563                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 564                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 565         },
 566         [MLX5_EXPANSION_IPV6] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 568                                                   MLX5_EXPANSION_IPV6_TCP),
 569                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 570                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 571                         ETH_RSS_NONFRAG_IPV6_OTHER,
 572         },
 573         [MLX5_EXPANSION_IPV6_UDP] = {
 574                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 575                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 576         },
 577         [MLX5_EXPANSION_IPV6_TCP] = {
 578                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 579                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 580         },
 581 };
 582
 583 static struct rte_flow_shared_action *
 584 mlx5_shared_action_create(struct rte_eth_dev *dev,
 585                           const struct rte_flow_shared_action_conf *conf,
 586                           const struct rte_flow_action *action,
 587                           struct rte_flow_error *error);
 588 static int mlx5_shared_action_destroy
 589                                 (struct rte_eth_dev *dev,
 590                                  struct rte_flow_shared_action *shared_action,
 591                                  struct rte_flow_error *error);
 592 static int mlx5_shared_action_update
 593                                 (struct rte_eth_dev *dev,
 594                                  struct rte_flow_shared_action *shared_action,
 595                                  const struct rte_flow_action *action,
 596                                  struct rte_flow_error *error);
 597 static int mlx5_shared_action_query
 598                                 (struct rte_eth_dev *dev,
 599                                  const struct rte_flow_shared_action *action,
 600                                  void *data,
 601                                  struct rte_flow_error *error);
 602 static int
 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 604                     struct rte_flow_tunnel *app_tunnel,
 605                     struct rte_flow_action **actions,
 606                     uint32_t *num_of_actions,
 607                     struct rte_flow_error *error);
 608 static int
 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 610                        struct rte_flow_tunnel *app_tunnel,
 611                        struct rte_flow_item **items,
 612                        uint32_t *num_of_items,
 613                        struct rte_flow_error *error);
 614 static int
 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 616                               struct rte_flow_item *pmd_items,
 617                               uint32_t num_items, struct rte_flow_error *err);
 618 static int
 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 620                                 struct rte_flow_action *pmd_actions,
 621                                 uint32_t num_actions,
 622                                 struct rte_flow_error *err);
 623 static int
 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 625                                   struct rte_mbuf *m,
 626                                   struct rte_flow_restore_info *info,
 627                                   struct rte_flow_error *err);
 628
 629 static const struct rte_flow_ops mlx5_flow_ops = {
 630         .validate = mlx5_flow_validate,
 631         .create = mlx5_flow_create,
 632         .destroy = mlx5_flow_destroy,
 633         .flush = mlx5_flow_flush,
 634         .isolate = mlx5_flow_isolate,
 635         .query = mlx5_flow_query,
 636         .dev_dump = mlx5_flow_dev_dump,
 637         .get_aged_flows = mlx5_flow_get_aged_flows,
 638         .shared_action_create = mlx5_shared_action_create,
 639         .shared_action_destroy = mlx5_shared_action_destroy,
 640         .shared_action_update = mlx5_shared_action_update,
 641         .shared_action_query = mlx5_shared_action_query,
 642         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 643         .tunnel_match = mlx5_flow_tunnel_match,
 644         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 645         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 646         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 647 };
 648
 649 /* Tunnel information. */
 650 struct mlx5_flow_tunnel_info {
 651         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 652         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 653 };
 654
 655 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 656         {
 657                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 658                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 659         },
 660         {
 661                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 662                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 663         },
 664         {
 665                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 666                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 667         },
 668         {
 669                 .tunnel = MLX5_FLOW_LAYER_GRE,
 670                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 671         },
 672         {
 673                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 674                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 675         },
 676         {
 677                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 678                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 679         },
 680         {
 681                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 682                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 683         },
 684         {
 685                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 686                 .ptype = RTE_PTYPE_TUNNEL_IP,
 687         },
 688         {
 689                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 690                 .ptype = RTE_PTYPE_TUNNEL_IP,
 691         },
 692         {
 693                 .tunnel = MLX5_FLOW_LAYER_GTP,
 694                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 695         },
 696 };
 697
 698 /* Key of thread specific flow workspace data. */
 699 static pthread_key_t key_workspace;
 700
 701 /* Thread specific flow workspace data once initialization data. */
 702 static pthread_once_t key_workspace_init;
 703
 704
 705 /**
 706  * Translate tag ID to register.
 707  *
 708  * @param[in] dev
 709  *   Pointer to the Ethernet device structure.
 710  * @param[in] feature
 711  *   The feature that request the register.
 712  * @param[in] id
 713  *   The request register ID.
 714  * @param[out] error
 715  *   Error description in case of any.
 716  *
 717  * @return
 718  *   The request register on success, a negative errno
 719  *   value otherwise and rte_errno is set.
 720  */
 721 int
 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 723                      enum mlx5_feature_name feature,
 724                      uint32_t id,
 725                      struct rte_flow_error *error)
 726 {
 727         struct mlx5_priv *priv = dev->data->dev_private;
 728         struct mlx5_dev_config *config = &priv->config;
 729         enum modify_reg start_reg;
 730         bool skip_mtr_reg = false;
 731
 732         switch (feature) {
 733         case MLX5_HAIRPIN_RX:
 734                 return REG_B;
 735         case MLX5_HAIRPIN_TX:
 736                 return REG_A;
 737         case MLX5_METADATA_RX:
 738                 switch (config->dv_xmeta_en) {
 739                 case MLX5_XMETA_MODE_LEGACY:
 740                         return REG_B;
 741                 case MLX5_XMETA_MODE_META16:
 742                         return REG_C_0;
 743                 case MLX5_XMETA_MODE_META32:
 744                         return REG_C_1;
 745                 }
 746                 break;
 747         case MLX5_METADATA_TX:
 748                 return REG_A;
 749         case MLX5_METADATA_FDB:
 750                 switch (config->dv_xmeta_en) {
 751                 case MLX5_XMETA_MODE_LEGACY:
 752                         return REG_NON;
 753                 case MLX5_XMETA_MODE_META16:
 754                         return REG_C_0;
 755                 case MLX5_XMETA_MODE_META32:
 756                         return REG_C_1;
 757                 }
 758                 break;
 759         case MLX5_FLOW_MARK:
 760                 switch (config->dv_xmeta_en) {
 761                 case MLX5_XMETA_MODE_LEGACY:
 762                         return REG_NON;
 763                 case MLX5_XMETA_MODE_META16:
 764                         return REG_C_1;
 765                 case MLX5_XMETA_MODE_META32:
 766                         return REG_C_0;
 767                 }
 768                 break;
 769         case MLX5_MTR_SFX:
 770                 /*
 771                  * If meter color and flow match share one register, flow match
 772                  * should use the meter color register for match.
 773                  */
 774                 if (priv->mtr_reg_share)
 775                         return priv->mtr_color_reg;
 776                 else
 777                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 778                                REG_C_3;
 779         case MLX5_MTR_COLOR:
 780         case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */
 781                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 782                 return priv->mtr_color_reg;
 783         case MLX5_COPY_MARK:
 784                 /*
 785                  * Metadata COPY_MARK register using is in meter suffix sub
 786                  * flow while with meter. It's safe to share the same register.
 787                  */
 788                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 789         case MLX5_APP_TAG:
 790                 /*
 791                  * If meter is enable, it will engage the register for color
 792                  * match and flow match. If meter color match is not using the
 793                  * REG_C_2, need to skip the REG_C_x be used by meter color
 794                  * match.
 795                  * If meter is disable, free to use all available registers.
 796                  */
 797                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 798                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 799                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 800                 if (id > (REG_C_7 - start_reg))
 801                         return rte_flow_error_set(error, EINVAL,
 802                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 803                                                   NULL, "invalid tag id");
 804                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 805                         return rte_flow_error_set(error, ENOTSUP,
 806                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 807                                                   NULL, "unsupported tag id");
 808                 /*
 809                  * This case means meter is using the REG_C_x great than 2.
 810                  * Take care not to conflict with meter color REG_C_x.
 811                  * If the available index REG_C_y >= REG_C_x, skip the
 812                  * color register.
 813                  */
 814                 if (skip_mtr_reg && config->flow_mreg_c
 815                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 816                         if (id >= (REG_C_7 - start_reg))
 817                                 return rte_flow_error_set(error, EINVAL,
 818                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 819                                                         NULL, "invalid tag id");
 820                         if (config->flow_mreg_c
 821                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 822                                 return config->flow_mreg_c
 823                                                [id + 1 + start_reg - REG_C_0];
 824                         return rte_flow_error_set(error, ENOTSUP,
 825                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 826                                                   NULL, "unsupported tag id");
 827                 }
 828                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 829         }
 830         MLX5_ASSERT(false);
 831         return rte_flow_error_set(error, EINVAL,
 832                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 833                                   NULL, "invalid feature name");
 834 }
 835
 836 /**
 837  * Check extensive flow metadata register support.
 838  *
 839  * @param dev
 840  *   Pointer to rte_eth_dev structure.
 841  *
 842  * @return
 843  *   True if device supports extensive flow metadata register, otherwise false.
 844  */
 845 bool
 846 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 847 {
 848         struct mlx5_priv *priv = dev->data->dev_private;
 849         struct mlx5_dev_config *config = &priv->config;
 850
 851         /*
 852          * Having available reg_c can be regarded inclusively as supporting
 853          * extensive flow metadata register, which could mean,
 854          * - metadata register copy action by modify header.
 855          * - 16 modify header actions is supported.
 856          * - reg_c's are preserved across different domain (FDB and NIC) on
 857          *   packet loopback by flow lookup miss.
 858          */
 859         return config->flow_mreg_c[2] != REG_NON;
 860 }
 861
 862 /**
 863  * Verify the @p item specifications (spec, last, mask) are compatible with the
 864  * NIC capabilities.
 865  *
 866  * @param[in] item
 867  *   Item specification.
 868  * @param[in] mask
 869  *   @p item->mask or flow default bit-masks.
 870  * @param[in] nic_mask
 871  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 872  * @param[in] size
 873  *   Bit-masks size in bytes.
 874  * @param[in] range_accepted
 875  *   True if range of values is accepted for specific fields, false otherwise.
 876  * @param[out] error
 877  *   Pointer to error structure.
 878  *
 879  * @return
 880  *   0 on success, a negative errno value otherwise and rte_errno is set.
 881  */
 882 int
 883 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 884                           const uint8_t *mask,
 885                           const uint8_t *nic_mask,
 886                           unsigned int size,
 887                           bool range_accepted,
 888                           struct rte_flow_error *error)
 889 {
 890         unsigned int i;
 891
 892         MLX5_ASSERT(nic_mask);
 893         for (i = 0; i < size; ++i)
 894                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 895                         return rte_flow_error_set(error, ENOTSUP,
 896                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 897                                                   item,
 898                                                   "mask enables non supported"
 899                                                   " bits");
 900         if (!item->spec && (item->mask || item->last))
 901                 return rte_flow_error_set(error, EINVAL,
 902                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 903                                           "mask/last without a spec is not"
 904                                           " supported");
 905         if (item->spec && item->last && !range_accepted) {
 906                 uint8_t spec[size];
 907                 uint8_t last[size];
 908                 unsigned int i;
 909                 int ret;
 910
 911                 for (i = 0; i < size; ++i) {
 912                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 913                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 914                 }
 915                 ret = memcmp(spec, last, size);
 916                 if (ret != 0)
 917                         return rte_flow_error_set(error, EINVAL,
 918                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 919                                                   item,
 920                                                   "range is not valid");
 921         }
 922         return 0;
 923 }
 924
 925 /**
 926  * Adjust the hash fields according to the @p flow information.
 927  *
 928  * @param[in] dev_flow.
 929  *   Pointer to the mlx5_flow.
 930  * @param[in] tunnel
 931  *   1 when the hash field is for a tunnel item.
 932  * @param[in] layer_types
 933  *   ETH_RSS_* types.
 934  * @param[in] hash_fields
 935  *   Item hash fields.
 936  *
 937  * @return
 938  *   The hash fields that should be used.
 939  */
 940 uint64_t
 941 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 942                             int tunnel __rte_unused, uint64_t layer_types,
 943                             uint64_t hash_fields)
 944 {
 945 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 946         int rss_request_inner = rss_desc->level >= 2;
 947
 948         /* Check RSS hash level for tunnel. */
 949         if (tunnel && rss_request_inner)
 950                 hash_fields |= IBV_RX_HASH_INNER;
 951         else if (tunnel || rss_request_inner)
 952                 return 0;
 953 #endif
 954         /* Check if requested layer matches RSS hash fields. */
 955         if (!(rss_desc->types & layer_types))
 956                 return 0;
 957         return hash_fields;
 958 }
 959
 960 /**
 961  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 962  * if several tunnel rules are used on this queue, the tunnel ptype will be
 963  * cleared.
 964  *
 965  * @param rxq_ctrl
 966  *   Rx queue to update.
 967  */
 968 static void
 969 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 970 {
 971         unsigned int i;
 972         uint32_t tunnel_ptype = 0;
 973
 974         /* Look up for the ptype to use. */
 975         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 976                 if (!rxq_ctrl->flow_tunnels_n[i])
 977                         continue;
 978                 if (!tunnel_ptype) {
 979                         tunnel_ptype = tunnels_info[i].ptype;
 980                 } else {
 981                         tunnel_ptype = 0;
 982                         break;
 983                 }
 984         }
 985         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 986 }
 987
 988 /**
 989  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 990  * flow.
 991  *
 992  * @param[in] dev
 993  *   Pointer to the Ethernet device structure.
 994  * @param[in] dev_handle
 995  *   Pointer to device flow handle structure.
 996  */
 997 static void
 998 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
 999                        struct mlx5_flow_handle *dev_handle)
1000 {
1001         struct mlx5_priv *priv = dev->data->dev_private;
1002         const int mark = dev_handle->mark;
1003         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1004         struct mlx5_hrxq *hrxq;
1005         unsigned int i;
1006
1007         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1008                 return;
1009         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1010                               dev_handle->rix_hrxq);
1011         if (!hrxq)
1012                 return;
1013         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1014                 int idx = hrxq->ind_table->queues[i];
1015                 struct mlx5_rxq_ctrl *rxq_ctrl =
1016                         container_of((*priv->rxqs)[idx],
1017                                      struct mlx5_rxq_ctrl, rxq);
1018
1019                 /*
1020                  * To support metadata register copy on Tx loopback,
1021                  * this must be always enabled (metadata may arive
1022                  * from other port - not from local flows only.
1023                  */
1024                 if (priv->config.dv_flow_en &&
1025                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1026                     mlx5_flow_ext_mreg_supported(dev)) {
1027                         rxq_ctrl->rxq.mark = 1;
1028                         rxq_ctrl->flow_mark_n = 1;
1029                 } else if (mark) {
1030                         rxq_ctrl->rxq.mark = 1;
1031                         rxq_ctrl->flow_mark_n++;
1032                 }
1033                 if (tunnel) {
1034                         unsigned int j;
1035
1036                         /* Increase the counter matching the flow. */
1037                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1038                                 if ((tunnels_info[j].tunnel &
1039                                      dev_handle->layers) ==
1040                                     tunnels_info[j].tunnel) {
1041                                         rxq_ctrl->flow_tunnels_n[j]++;
1042                                         break;
1043                                 }
1044                         }
1045                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1046                 }
1047         }
1048 }
1049
1050 /**
1051  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1052  *
1053  * @param[in] dev
1054  *   Pointer to the Ethernet device structure.
1055  * @param[in] flow
1056  *   Pointer to flow structure.
1057  */
1058 static void
1059 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1060 {
1061         struct mlx5_priv *priv = dev->data->dev_private;
1062         uint32_t handle_idx;
1063         struct mlx5_flow_handle *dev_handle;
1064
1065         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1066                        handle_idx, dev_handle, next)
1067                 flow_drv_rxq_flags_set(dev, dev_handle);
1068 }
1069
1070 /**
1071  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1072  * device flow if no other flow uses it with the same kind of request.
1073  *
1074  * @param dev
1075  *   Pointer to Ethernet device.
1076  * @param[in] dev_handle
1077  *   Pointer to the device flow handle structure.
1078  */
1079 static void
1080 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1081                         struct mlx5_flow_handle *dev_handle)
1082 {
1083         struct mlx5_priv *priv = dev->data->dev_private;
1084         const int mark = dev_handle->mark;
1085         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1086         struct mlx5_hrxq *hrxq;
1087         unsigned int i;
1088
1089         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1090                 return;
1091         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1092                               dev_handle->rix_hrxq);
1093         if (!hrxq)
1094                 return;
1095         MLX5_ASSERT(dev->data->dev_started);
1096         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1097                 int idx = hrxq->ind_table->queues[i];
1098                 struct mlx5_rxq_ctrl *rxq_ctrl =
1099                         container_of((*priv->rxqs)[idx],
1100                                      struct mlx5_rxq_ctrl, rxq);
1101
1102                 if (priv->config.dv_flow_en &&
1103                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1104                     mlx5_flow_ext_mreg_supported(dev)) {
1105                         rxq_ctrl->rxq.mark = 1;
1106                         rxq_ctrl->flow_mark_n = 1;
1107                 } else if (mark) {
1108                         rxq_ctrl->flow_mark_n--;
1109                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1110                 }
1111                 if (tunnel) {
1112                         unsigned int j;
1113
1114                         /* Decrease the counter matching the flow. */
1115                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1116                                 if ((tunnels_info[j].tunnel &
1117                                      dev_handle->layers) ==
1118                                     tunnels_info[j].tunnel) {
1119                                         rxq_ctrl->flow_tunnels_n[j]--;
1120                                         break;
1121                                 }
1122                         }
1123                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1124                 }
1125         }
1126 }
1127
1128 /**
1129  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1130  * @p flow if no other flow uses it with the same kind of request.
1131  *
1132  * @param dev
1133  *   Pointer to Ethernet device.
1134  * @param[in] flow
1135  *   Pointer to the flow.
1136  */
1137 static void
1138 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1139 {
1140         struct mlx5_priv *priv = dev->data->dev_private;
1141         uint32_t handle_idx;
1142         struct mlx5_flow_handle *dev_handle;
1143
1144         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1145                        handle_idx, dev_handle, next)
1146                 flow_drv_rxq_flags_trim(dev, dev_handle);
1147 }
1148
1149 /**
1150  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1151  *
1152  * @param dev
1153  *   Pointer to Ethernet device.
1154  */
1155 static void
1156 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1157 {
1158         struct mlx5_priv *priv = dev->data->dev_private;
1159         unsigned int i;
1160
1161         for (i = 0; i != priv->rxqs_n; ++i) {
1162                 struct mlx5_rxq_ctrl *rxq_ctrl;
1163                 unsigned int j;
1164
1165                 if (!(*priv->rxqs)[i])
1166                         continue;
1167                 rxq_ctrl = container_of((*priv->rxqs)[i],
1168                                         struct mlx5_rxq_ctrl, rxq);
1169                 rxq_ctrl->flow_mark_n = 0;
1170                 rxq_ctrl->rxq.mark = 0;
1171                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1172                         rxq_ctrl->flow_tunnels_n[j] = 0;
1173                 rxq_ctrl->rxq.tunnel = 0;
1174         }
1175 }
1176
1177 /**
1178  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1179  *
1180  * @param[in] dev
1181  *   Pointer to the Ethernet device structure.
1182  */
1183 void
1184 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1185 {
1186         struct mlx5_priv *priv = dev->data->dev_private;
1187         struct mlx5_rxq_data *data;
1188         unsigned int i;
1189
1190         for (i = 0; i != priv->rxqs_n; ++i) {
1191                 if (!(*priv->rxqs)[i])
1192                         continue;
1193                 data = (*priv->rxqs)[i];
1194                 if (!rte_flow_dynf_metadata_avail()) {
1195                         data->dynf_meta = 0;
1196                         data->flow_meta_mask = 0;
1197                         data->flow_meta_offset = -1;
1198                 } else {
1199                         data->dynf_meta = 1;
1200                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1201                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1202                 }
1203         }
1204 }
1205
1206 /*
1207  * return a pointer to the desired action in the list of actions.
1208  *
1209  * @param[in] actions
1210  *   The list of actions to search the action in.
1211  * @param[in] action
1212  *   The action to find.
1213  *
1214  * @return
1215  *   Pointer to the action in the list, if found. NULL otherwise.
1216  */
1217 const struct rte_flow_action *
1218 mlx5_flow_find_action(const struct rte_flow_action *actions,
1219                       enum rte_flow_action_type action)
1220 {
1221         if (actions == NULL)
1222                 return NULL;
1223         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1224                 if (actions->type == action)
1225                         return actions;
1226         return NULL;
1227 }
1228
1229 /*
1230  * Validate the flag action.
1231  *
1232  * @param[in] action_flags
1233  *   Bit-fields that holds the actions detected until now.
1234  * @param[in] attr
1235  *   Attributes of flow that includes this action.
1236  * @param[out] error
1237  *   Pointer to error structure.
1238  *
1239  * @return
1240  *   0 on success, a negative errno value otherwise and rte_errno is set.
1241  */
1242 int
1243 mlx5_flow_validate_action_flag(uint64_t action_flags,
1244                                const struct rte_flow_attr *attr,
1245                                struct rte_flow_error *error)
1246 {
1247         if (action_flags & MLX5_FLOW_ACTION_MARK)
1248                 return rte_flow_error_set(error, EINVAL,
1249                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1250                                           "can't mark and flag in same flow");
1251         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1252                 return rte_flow_error_set(error, EINVAL,
1253                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1254                                           "can't have 2 flag"
1255                                           " actions in same flow");
1256         if (attr->egress)
1257                 return rte_flow_error_set(error, ENOTSUP,
1258                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1259                                           "flag action not supported for "
1260                                           "egress");
1261         return 0;
1262 }
1263
1264 /*
1265  * Validate the mark action.
1266  *
1267  * @param[in] action
1268  *   Pointer to the queue action.
1269  * @param[in] action_flags
1270  *   Bit-fields that holds the actions detected until now.
1271  * @param[in] attr
1272  *   Attributes of flow that includes this action.
1273  * @param[out] error
1274  *   Pointer to error structure.
1275  *
1276  * @return
1277  *   0 on success, a negative errno value otherwise and rte_errno is set.
1278  */
1279 int
1280 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1281                                uint64_t action_flags,
1282                                const struct rte_flow_attr *attr,
1283                                struct rte_flow_error *error)
1284 {
1285         const struct rte_flow_action_mark *mark = action->conf;
1286
1287         if (!mark)
1288                 return rte_flow_error_set(error, EINVAL,
1289                                           RTE_FLOW_ERROR_TYPE_ACTION,
1290                                           action,
1291                                           "configuration cannot be null");
1292         if (mark->id >= MLX5_FLOW_MARK_MAX)
1293                 return rte_flow_error_set(error, EINVAL,
1294                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1295                                           &mark->id,
1296                                           "mark id must in 0 <= id < "
1297                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1298         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1299                 return rte_flow_error_set(error, EINVAL,
1300                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1301                                           "can't flag and mark in same flow");
1302         if (action_flags & MLX5_FLOW_ACTION_MARK)
1303                 return rte_flow_error_set(error, EINVAL,
1304                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1305                                           "can't have 2 mark actions in same"
1306                                           " flow");
1307         if (attr->egress)
1308                 return rte_flow_error_set(error, ENOTSUP,
1309                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1310                                           "mark action not supported for "
1311                                           "egress");
1312         return 0;
1313 }
1314
1315 /*
1316  * Validate the drop action.
1317  *
1318  * @param[in] action_flags
1319  *   Bit-fields that holds the actions detected until now.
1320  * @param[in] attr
1321  *   Attributes of flow that includes this action.
1322  * @param[out] error
1323  *   Pointer to error structure.
1324  *
1325  * @return
1326  *   0 on success, a negative errno value otherwise and rte_errno is set.
1327  */
1328 int
1329 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1330                                const struct rte_flow_attr *attr,
1331                                struct rte_flow_error *error)
1332 {
1333         if (attr->egress)
1334                 return rte_flow_error_set(error, ENOTSUP,
1335                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1336                                           "drop action not supported for "
1337                                           "egress");
1338         return 0;
1339 }
1340
1341 /*
1342  * Validate the queue action.
1343  *
1344  * @param[in] action
1345  *   Pointer to the queue action.
1346  * @param[in] action_flags
1347  *   Bit-fields that holds the actions detected until now.
1348  * @param[in] dev
1349  *   Pointer to the Ethernet device structure.
1350  * @param[in] attr
1351  *   Attributes of flow that includes this action.
1352  * @param[out] error
1353  *   Pointer to error structure.
1354  *
1355  * @return
1356  *   0 on success, a negative errno value otherwise and rte_errno is set.
1357  */
1358 int
1359 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1360                                 uint64_t action_flags,
1361                                 struct rte_eth_dev *dev,
1362                                 const struct rte_flow_attr *attr,
1363                                 struct rte_flow_error *error)
1364 {
1365         struct mlx5_priv *priv = dev->data->dev_private;
1366         const struct rte_flow_action_queue *queue = action->conf;
1367
1368         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1369                 return rte_flow_error_set(error, EINVAL,
1370                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1371                                           "can't have 2 fate actions in"
1372                                           " same flow");
1373         if (!priv->rxqs_n)
1374                 return rte_flow_error_set(error, EINVAL,
1375                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1376                                           NULL, "No Rx queues configured");
1377         if (queue->index >= priv->rxqs_n)
1378                 return rte_flow_error_set(error, EINVAL,
1379                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1380                                           &queue->index,
1381                                           "queue index out of range");
1382         if (!(*priv->rxqs)[queue->index])
1383                 return rte_flow_error_set(error, EINVAL,
1384                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1385                                           &queue->index,
1386                                           "queue is not configured");
1387         if (attr->egress)
1388                 return rte_flow_error_set(error, ENOTSUP,
1389                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1390                                           "queue action not supported for "
1391                                           "egress");
1392         return 0;
1393 }
1394
1395 /*
1396  * Validate the rss action.
1397  *
1398  * @param[in] dev
1399  *   Pointer to the Ethernet device structure.
1400  * @param[in] action
1401  *   Pointer to the queue action.
1402  * @param[out] error
1403  *   Pointer to error structure.
1404  *
1405  * @return
1406  *   0 on success, a negative errno value otherwise and rte_errno is set.
1407  */
1408 int
1409 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1410                          const struct rte_flow_action *action,
1411                          struct rte_flow_error *error)
1412 {
1413         struct mlx5_priv *priv = dev->data->dev_private;
1414         const struct rte_flow_action_rss *rss = action->conf;
1415         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1416         unsigned int i;
1417
1418         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1419             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1420                 return rte_flow_error_set(error, ENOTSUP,
1421                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1422                                           &rss->func,
1423                                           "RSS hash function not supported");
1424 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1425         if (rss->level > 2)
1426 #else
1427         if (rss->level > 1)
1428 #endif
1429                 return rte_flow_error_set(error, ENOTSUP,
1430                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1431                                           &rss->level,
1432                                           "tunnel RSS is not supported");
1433         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1434         if (rss->key_len == 0 && rss->key != NULL)
1435                 return rte_flow_error_set(error, ENOTSUP,
1436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1437                                           &rss->key_len,
1438                                           "RSS hash key length 0");
1439         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1440                 return rte_flow_error_set(error, ENOTSUP,
1441                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1442                                           &rss->key_len,
1443                                           "RSS hash key too small");
1444         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1445                 return rte_flow_error_set(error, ENOTSUP,
1446                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1447                                           &rss->key_len,
1448                                           "RSS hash key too large");
1449         if (rss->queue_num > priv->config.ind_table_max_size)
1450                 return rte_flow_error_set(error, ENOTSUP,
1451                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1452                                           &rss->queue_num,
1453                                           "number of queues too large");
1454         if (rss->types & MLX5_RSS_HF_MASK)
1455                 return rte_flow_error_set(error, ENOTSUP,
1456                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1457                                           &rss->types,
1458                                           "some RSS protocols are not"
1459                                           " supported");
1460         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1461             !(rss->types & ETH_RSS_IP))
1462                 return rte_flow_error_set(error, EINVAL,
1463                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1464                                           "L3 partial RSS requested but L3 RSS"
1465                                           " type not specified");
1466         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1467             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1470                                           "L4 partial RSS requested but L4 RSS"
1471                                           " type not specified");
1472         if (!priv->rxqs_n)
1473                 return rte_flow_error_set(error, EINVAL,
1474                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1475                                           NULL, "No Rx queues configured");
1476         if (!rss->queue_num)
1477                 return rte_flow_error_set(error, EINVAL,
1478                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1479                                           NULL, "No queues configured");
1480         for (i = 0; i != rss->queue_num; ++i) {
1481                 struct mlx5_rxq_ctrl *rxq_ctrl;
1482
1483                 if (rss->queue[i] >= priv->rxqs_n)
1484                         return rte_flow_error_set
1485                                 (error, EINVAL,
1486                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1487                                  &rss->queue[i], "queue index out of range");
1488                 if (!(*priv->rxqs)[rss->queue[i]])
1489                         return rte_flow_error_set
1490                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1491                                  &rss->queue[i], "queue is not configured");
1492                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1493                                         struct mlx5_rxq_ctrl, rxq);
1494                 if (i == 0)
1495                         rxq_type = rxq_ctrl->type;
1496                 if (rxq_type != rxq_ctrl->type)
1497                         return rte_flow_error_set
1498                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1499                                  &rss->queue[i],
1500                                  "combining hairpin and regular RSS queues is not supported");
1501         }
1502         return 0;
1503 }
1504
1505 /*
1506  * Validate the rss action.
1507  *
1508  * @param[in] action
1509  *   Pointer to the queue action.
1510  * @param[in] action_flags
1511  *   Bit-fields that holds the actions detected until now.
1512  * @param[in] dev
1513  *   Pointer to the Ethernet device structure.
1514  * @param[in] attr
1515  *   Attributes of flow that includes this action.
1516  * @param[in] item_flags
1517  *   Items that were detected.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 int
1525 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1526                               uint64_t action_flags,
1527                               struct rte_eth_dev *dev,
1528                               const struct rte_flow_attr *attr,
1529                               uint64_t item_flags,
1530                               struct rte_flow_error *error)
1531 {
1532         const struct rte_flow_action_rss *rss = action->conf;
1533         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1534         int ret;
1535
1536         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1539                                           "can't have 2 fate actions"
1540                                           " in same flow");
1541         ret = mlx5_validate_action_rss(dev, action, error);
1542         if (ret)
1543                 return ret;
1544         if (attr->egress)
1545                 return rte_flow_error_set(error, ENOTSUP,
1546                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1547                                           "rss action not supported for "
1548                                           "egress");
1549         if (rss->level > 1 && !tunnel)
1550                 return rte_flow_error_set(error, EINVAL,
1551                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1552                                           "inner RSS is not supported for "
1553                                           "non-tunnel flows");
1554         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1555             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1556                 return rte_flow_error_set(error, EINVAL,
1557                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1558                                           "RSS on eCPRI is not supported now");
1559         }
1560         return 0;
1561 }
1562
1563 /*
1564  * Validate the default miss action.
1565  *
1566  * @param[in] action_flags
1567  *   Bit-fields that holds the actions detected until now.
1568  * @param[out] error
1569  *   Pointer to error structure.
1570  *
1571  * @return
1572  *   0 on success, a negative errno value otherwise and rte_errno is set.
1573  */
1574 int
1575 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1576                                 const struct rte_flow_attr *attr,
1577                                 struct rte_flow_error *error)
1578 {
1579         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1580                 return rte_flow_error_set(error, EINVAL,
1581                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1582                                           "can't have 2 fate actions in"
1583                                           " same flow");
1584         if (attr->egress)
1585                 return rte_flow_error_set(error, ENOTSUP,
1586                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1587                                           "default miss action not supported "
1588                                           "for egress");
1589         if (attr->group)
1590                 return rte_flow_error_set(error, ENOTSUP,
1591                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1592                                           "only group 0 is supported");
1593         if (attr->transfer)
1594                 return rte_flow_error_set(error, ENOTSUP,
1595                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1596                                           NULL, "transfer is not supported");
1597         return 0;
1598 }
1599
1600 /*
1601  * Validate the count action.
1602  *
1603  * @param[in] dev
1604  *   Pointer to the Ethernet device structure.
1605  * @param[in] attr
1606  *   Attributes of flow that includes this action.
1607  * @param[out] error
1608  *   Pointer to error structure.
1609  *
1610  * @return
1611  *   0 on success, a negative errno value otherwise and rte_errno is set.
1612  */
1613 int
1614 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1615                                 const struct rte_flow_attr *attr,
1616                                 struct rte_flow_error *error)
1617 {
1618         if (attr->egress)
1619                 return rte_flow_error_set(error, ENOTSUP,
1620                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1621                                           "count action not supported for "
1622                                           "egress");
1623         return 0;
1624 }
1625
1626 /**
1627  * Verify the @p attributes will be correctly understood by the NIC and store
1628  * them in the @p flow if everything is correct.
1629  *
1630  * @param[in] dev
1631  *   Pointer to the Ethernet device structure.
1632  * @param[in] attributes
1633  *   Pointer to flow attributes
1634  * @param[out] error
1635  *   Pointer to error structure.
1636  *
1637  * @return
1638  *   0 on success, a negative errno value otherwise and rte_errno is set.
1639  */
1640 int
1641 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1642                               const struct rte_flow_attr *attributes,
1643                               struct rte_flow_error *error)
1644 {
1645         struct mlx5_priv *priv = dev->data->dev_private;
1646         uint32_t priority_max = priv->config.flow_prio - 1;
1647
1648         if (attributes->group)
1649                 return rte_flow_error_set(error, ENOTSUP,
1650                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1651                                           NULL, "groups is not supported");
1652         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1653             attributes->priority >= priority_max)
1654                 return rte_flow_error_set(error, ENOTSUP,
1655                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1656                                           NULL, "priority out of range");
1657         if (attributes->egress)
1658                 return rte_flow_error_set(error, ENOTSUP,
1659                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1660                                           "egress is not supported");
1661         if (attributes->transfer && !priv->config.dv_esw_en)
1662                 return rte_flow_error_set(error, ENOTSUP,
1663                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1664                                           NULL, "transfer is not supported");
1665         if (!attributes->ingress)
1666                 return rte_flow_error_set(error, EINVAL,
1667                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1668                                           NULL,
1669                                           "ingress attribute is mandatory");
1670         return 0;
1671 }
1672
1673 /**
1674  * Validate ICMP6 item.
1675  *
1676  * @param[in] item
1677  *   Item specification.
1678  * @param[in] item_flags
1679  *   Bit-fields that holds the items detected until now.
1680  * @param[in] ext_vlan_sup
1681  *   Whether extended VLAN features are supported or not.
1682  * @param[out] error
1683  *   Pointer to error structure.
1684  *
1685  * @return
1686  *   0 on success, a negative errno value otherwise and rte_errno is set.
1687  */
1688 int
1689 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1690                                uint64_t item_flags,
1691                                uint8_t target_protocol,
1692                                struct rte_flow_error *error)
1693 {
1694         const struct rte_flow_item_icmp6 *mask = item->mask;
1695         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1696         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1697                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1698         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1699                                       MLX5_FLOW_LAYER_OUTER_L4;
1700         int ret;
1701
1702         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1703                 return rte_flow_error_set(error, EINVAL,
1704                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1705                                           "protocol filtering not compatible"
1706                                           " with ICMP6 layer");
1707         if (!(item_flags & l3m))
1708                 return rte_flow_error_set(error, EINVAL,
1709                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1710                                           "IPv6 is mandatory to filter on"
1711                                           " ICMP6");
1712         if (item_flags & l4m)
1713                 return rte_flow_error_set(error, EINVAL,
1714                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1715                                           "multiple L4 layers not supported");
1716         if (!mask)
1717                 mask = &rte_flow_item_icmp6_mask;
1718         ret = mlx5_flow_item_acceptable
1719                 (item, (const uint8_t *)mask,
1720                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1721                  sizeof(struct rte_flow_item_icmp6),
1722                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1723         if (ret < 0)
1724                 return ret;
1725         return 0;
1726 }
1727
1728 /**
1729  * Validate ICMP item.
1730  *
1731  * @param[in] item
1732  *   Item specification.
1733  * @param[in] item_flags
1734  *   Bit-fields that holds the items detected until now.
1735  * @param[out] error
1736  *   Pointer to error structure.
1737  *
1738  * @return
1739  *   0 on success, a negative errno value otherwise and rte_errno is set.
1740  */
1741 int
1742 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1743                              uint64_t item_flags,
1744                              uint8_t target_protocol,
1745                              struct rte_flow_error *error)
1746 {
1747         const struct rte_flow_item_icmp *mask = item->mask;
1748         const struct rte_flow_item_icmp nic_mask = {
1749                 .hdr.icmp_type = 0xff,
1750                 .hdr.icmp_code = 0xff,
1751                 .hdr.icmp_ident = RTE_BE16(0xffff),
1752                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1753         };
1754         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1755         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1756                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1757         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1758                                       MLX5_FLOW_LAYER_OUTER_L4;
1759         int ret;
1760
1761         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1762                 return rte_flow_error_set(error, EINVAL,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1764                                           "protocol filtering not compatible"
1765                                           " with ICMP layer");
1766         if (!(item_flags & l3m))
1767                 return rte_flow_error_set(error, EINVAL,
1768                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1769                                           "IPv4 is mandatory to filter"
1770                                           " on ICMP");
1771         if (item_flags & l4m)
1772                 return rte_flow_error_set(error, EINVAL,
1773                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1774                                           "multiple L4 layers not supported");
1775         if (!mask)
1776                 mask = &nic_mask;
1777         ret = mlx5_flow_item_acceptable
1778                 (item, (const uint8_t *)mask,
1779                  (const uint8_t *)&nic_mask,
1780                  sizeof(struct rte_flow_item_icmp),
1781                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1782         if (ret < 0)
1783                 return ret;
1784         return 0;
1785 }
1786
1787 /**
1788  * Validate Ethernet item.
1789  *
1790  * @param[in] item
1791  *   Item specification.
1792  * @param[in] item_flags
1793  *   Bit-fields that holds the items detected until now.
1794  * @param[out] error
1795  *   Pointer to error structure.
1796  *
1797  * @return
1798  *   0 on success, a negative errno value otherwise and rte_errno is set.
1799  */
1800 int
1801 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1802                             uint64_t item_flags, bool ext_vlan_sup,
1803                             struct rte_flow_error *error)
1804 {
1805         const struct rte_flow_item_eth *mask = item->mask;
1806         const struct rte_flow_item_eth nic_mask = {
1807                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1808                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1809                 .type = RTE_BE16(0xffff),
1810                 .has_vlan = ext_vlan_sup ? 1 : 0,
1811         };
1812         int ret;
1813         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1814         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1815                                        MLX5_FLOW_LAYER_OUTER_L2;
1816
1817         if (item_flags & ethm)
1818                 return rte_flow_error_set(error, ENOTSUP,
1819                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1820                                           "multiple L2 layers not supported");
1821         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1822             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1823                 return rte_flow_error_set(error, EINVAL,
1824                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1825                                           "L2 layer should not follow "
1826                                           "L3 layers");
1827         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1828             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1829                 return rte_flow_error_set(error, EINVAL,
1830                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1831                                           "L2 layer should not follow VLAN");
1832         if (!mask)
1833                 mask = &rte_flow_item_eth_mask;
1834         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1835                                         (const uint8_t *)&nic_mask,
1836                                         sizeof(struct rte_flow_item_eth),
1837                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1838         return ret;
1839 }
1840
1841 /**
1842  * Validate VLAN item.
1843  *
1844  * @param[in] item
1845  *   Item specification.
1846  * @param[in] item_flags
1847  *   Bit-fields that holds the items detected until now.
1848  * @param[in] dev
1849  *   Ethernet device flow is being created on.
1850  * @param[out] error
1851  *   Pointer to error structure.
1852  *
1853  * @return
1854  *   0 on success, a negative errno value otherwise and rte_errno is set.
1855  */
1856 int
1857 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1858                              uint64_t item_flags,
1859                              struct rte_eth_dev *dev,
1860                              struct rte_flow_error *error)
1861 {
1862         const struct rte_flow_item_vlan *spec = item->spec;
1863         const struct rte_flow_item_vlan *mask = item->mask;
1864         const struct rte_flow_item_vlan nic_mask = {
1865                 .tci = RTE_BE16(UINT16_MAX),
1866                 .inner_type = RTE_BE16(UINT16_MAX),
1867         };
1868         uint16_t vlan_tag = 0;
1869         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1870         int ret;
1871         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1872                                         MLX5_FLOW_LAYER_INNER_L4) :
1873                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1874                                         MLX5_FLOW_LAYER_OUTER_L4);
1875         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1876                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1877
1878         if (item_flags & vlanm)
1879                 return rte_flow_error_set(error, EINVAL,
1880                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1881                                           "multiple VLAN layers not supported");
1882         else if ((item_flags & l34m) != 0)
1883                 return rte_flow_error_set(error, EINVAL,
1884                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1885                                           "VLAN cannot follow L3/L4 layer");
1886         if (!mask)
1887                 mask = &rte_flow_item_vlan_mask;
1888         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1889                                         (const uint8_t *)&nic_mask,
1890                                         sizeof(struct rte_flow_item_vlan),
1891                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1892         if (ret)
1893                 return ret;
1894         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1895                 struct mlx5_priv *priv = dev->data->dev_private;
1896
1897                 if (priv->vmwa_context) {
1898                         /*
1899                          * Non-NULL context means we have a virtual machine
1900                          * and SR-IOV enabled, we have to create VLAN interface
1901                          * to make hypervisor to setup E-Switch vport
1902                          * context correctly. We avoid creating the multiple
1903                          * VLAN interfaces, so we cannot support VLAN tag mask.
1904                          */
1905                         return rte_flow_error_set(error, EINVAL,
1906                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1907                                                   item,
1908                                                   "VLAN tag mask is not"
1909                                                   " supported in virtual"
1910                                                   " environment");
1911                 }
1912         }
1913         if (spec) {
1914                 vlan_tag = spec->tci;
1915                 vlan_tag &= mask->tci;
1916         }
1917         /*
1918          * From verbs perspective an empty VLAN is equivalent
1919          * to a packet without VLAN layer.
1920          */
1921         if (!vlan_tag)
1922                 return rte_flow_error_set(error, EINVAL,
1923                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1924                                           item->spec,
1925                                           "VLAN cannot be empty");
1926         return 0;
1927 }
1928
1929 /**
1930  * Validate IPV4 item.
1931  *
1932  * @param[in] item
1933  *   Item specification.
1934  * @param[in] item_flags
1935  *   Bit-fields that holds the items detected until now.
1936  * @param[in] last_item
1937  *   Previous validated item in the pattern items.
1938  * @param[in] ether_type
1939  *   Type in the ethernet layer header (including dot1q).
1940  * @param[in] acc_mask
1941  *   Acceptable mask, if NULL default internal default mask
1942  *   will be used to check whether item fields are supported.
1943  * @param[in] range_accepted
1944  *   True if range of values is accepted for specific fields, false otherwise.
1945  * @param[out] error
1946  *   Pointer to error structure.
1947  *
1948  * @return
1949  *   0 on success, a negative errno value otherwise and rte_errno is set.
1950  */
1951 int
1952 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1953                              uint64_t item_flags,
1954                              uint64_t last_item,
1955                              uint16_t ether_type,
1956                              const struct rte_flow_item_ipv4 *acc_mask,
1957                              bool range_accepted,
1958                              struct rte_flow_error *error)
1959 {
1960         const struct rte_flow_item_ipv4 *mask = item->mask;
1961         const struct rte_flow_item_ipv4 *spec = item->spec;
1962         const struct rte_flow_item_ipv4 nic_mask = {
1963                 .hdr = {
1964                         .src_addr = RTE_BE32(0xffffffff),
1965                         .dst_addr = RTE_BE32(0xffffffff),
1966                         .type_of_service = 0xff,
1967                         .next_proto_id = 0xff,
1968                 },
1969         };
1970         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1971         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1972                                       MLX5_FLOW_LAYER_OUTER_L3;
1973         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1974                                       MLX5_FLOW_LAYER_OUTER_L4;
1975         int ret;
1976         uint8_t next_proto = 0xFF;
1977         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1978                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1979                                   MLX5_FLOW_LAYER_INNER_VLAN);
1980
1981         if ((last_item & l2_vlan) && ether_type &&
1982             ether_type != RTE_ETHER_TYPE_IPV4)
1983                 return rte_flow_error_set(error, EINVAL,
1984                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1985                                           "IPv4 cannot follow L2/VLAN layer "
1986                                           "which ether type is not IPv4");
1987         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1988                 if (mask && spec)
1989                         next_proto = mask->hdr.next_proto_id &
1990                                      spec->hdr.next_proto_id;
1991                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1992                         return rte_flow_error_set(error, EINVAL,
1993                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1994                                                   item,
1995                                                   "multiple tunnel "
1996                                                   "not supported");
1997         }
1998         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1999                 return rte_flow_error_set(error, EINVAL,
2000                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2001                                           "wrong tunnel type - IPv6 specified "
2002                                           "but IPv4 item provided");
2003         if (item_flags & l3m)
2004                 return rte_flow_error_set(error, ENOTSUP,
2005                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2006                                           "multiple L3 layers not supported");
2007         else if (item_flags & l4m)
2008                 return rte_flow_error_set(error, EINVAL,
2009                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2010                                           "L3 cannot follow an L4 layer.");
2011         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2012                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2013                 return rte_flow_error_set(error, EINVAL,
2014                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2015                                           "L3 cannot follow an NVGRE layer.");
2016         if (!mask)
2017                 mask = &rte_flow_item_ipv4_mask;
2018         else if (mask->hdr.next_proto_id != 0 &&
2019                  mask->hdr.next_proto_id != 0xff)
2020                 return rte_flow_error_set(error, EINVAL,
2021                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2022                                           "partial mask is not supported"
2023                                           " for protocol");
2024         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2025                                         acc_mask ? (const uint8_t *)acc_mask
2026                                                  : (const uint8_t *)&nic_mask,
2027                                         sizeof(struct rte_flow_item_ipv4),
2028                                         range_accepted, error);
2029         if (ret < 0)
2030                 return ret;
2031         return 0;
2032 }
2033
2034 /**
2035  * Validate IPV6 item.
2036  *
2037  * @param[in] item
2038  *   Item specification.
2039  * @param[in] item_flags
2040  *   Bit-fields that holds the items detected until now.
2041  * @param[in] last_item
2042  *   Previous validated item in the pattern items.
2043  * @param[in] ether_type
2044  *   Type in the ethernet layer header (including dot1q).
2045  * @param[in] acc_mask
2046  *   Acceptable mask, if NULL default internal default mask
2047  *   will be used to check whether item fields are supported.
2048  * @param[out] error
2049  *   Pointer to error structure.
2050  *
2051  * @return
2052  *   0 on success, a negative errno value otherwise and rte_errno is set.
2053  */
2054 int
2055 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2056                              uint64_t item_flags,
2057                              uint64_t last_item,
2058                              uint16_t ether_type,
2059                              const struct rte_flow_item_ipv6 *acc_mask,
2060                              struct rte_flow_error *error)
2061 {
2062         const struct rte_flow_item_ipv6 *mask = item->mask;
2063         const struct rte_flow_item_ipv6 *spec = item->spec;
2064         const struct rte_flow_item_ipv6 nic_mask = {
2065                 .hdr = {
2066                         .src_addr =
2067                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2068                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2069                         .dst_addr =
2070                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2071                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2072                         .vtc_flow = RTE_BE32(0xffffffff),
2073                         .proto = 0xff,
2074                 },
2075         };
2076         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2077         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2078                                       MLX5_FLOW_LAYER_OUTER_L3;
2079         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2080                                       MLX5_FLOW_LAYER_OUTER_L4;
2081         int ret;
2082         uint8_t next_proto = 0xFF;
2083         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2084                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2085                                   MLX5_FLOW_LAYER_INNER_VLAN);
2086
2087         if ((last_item & l2_vlan) && ether_type &&
2088             ether_type != RTE_ETHER_TYPE_IPV6)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2091                                           "IPv6 cannot follow L2/VLAN layer "
2092                                           "which ether type is not IPv6");
2093         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2094                 next_proto = spec->hdr.proto;
2095         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2096                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2097                         return rte_flow_error_set(error, EINVAL,
2098                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2099                                                   item,
2100                                                   "multiple tunnel "
2101                                                   "not supported");
2102         }
2103         if (next_proto == IPPROTO_HOPOPTS  ||
2104             next_proto == IPPROTO_ROUTING  ||
2105             next_proto == IPPROTO_FRAGMENT ||
2106             next_proto == IPPROTO_ESP      ||
2107             next_proto == IPPROTO_AH       ||
2108             next_proto == IPPROTO_DSTOPTS)
2109                 return rte_flow_error_set(error, EINVAL,
2110                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2111                                           "IPv6 proto (next header) should "
2112                                           "not be set as extension header");
2113         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2114                 return rte_flow_error_set(error, EINVAL,
2115                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2116                                           "wrong tunnel type - IPv4 specified "
2117                                           "but IPv6 item provided");
2118         if (item_flags & l3m)
2119                 return rte_flow_error_set(error, ENOTSUP,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2121                                           "multiple L3 layers not supported");
2122         else if (item_flags & l4m)
2123                 return rte_flow_error_set(error, EINVAL,
2124                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2125                                           "L3 cannot follow an L4 layer.");
2126         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2127                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2128                 return rte_flow_error_set(error, EINVAL,
2129                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2130                                           "L3 cannot follow an NVGRE layer.");
2131         if (!mask)
2132                 mask = &rte_flow_item_ipv6_mask;
2133         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2134                                         acc_mask ? (const uint8_t *)acc_mask
2135                                                  : (const uint8_t *)&nic_mask,
2136                                         sizeof(struct rte_flow_item_ipv6),
2137                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2138         if (ret < 0)
2139                 return ret;
2140         return 0;
2141 }
2142
2143 /**
2144  * Validate UDP item.
2145  *
2146  * @param[in] item
2147  *   Item specification.
2148  * @param[in] item_flags
2149  *   Bit-fields that holds the items detected until now.
2150  * @param[in] target_protocol
2151  *   The next protocol in the previous item.
2152  * @param[in] flow_mask
2153  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2154  * @param[out] error
2155  *   Pointer to error structure.
2156  *
2157  * @return
2158  *   0 on success, a negative errno value otherwise and rte_errno is set.
2159  */
2160 int
2161 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2162                             uint64_t item_flags,
2163                             uint8_t target_protocol,
2164                             struct rte_flow_error *error)
2165 {
2166         const struct rte_flow_item_udp *mask = item->mask;
2167         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2168         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2169                                       MLX5_FLOW_LAYER_OUTER_L3;
2170         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2171                                       MLX5_FLOW_LAYER_OUTER_L4;
2172         int ret;
2173
2174         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2175                 return rte_flow_error_set(error, EINVAL,
2176                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2177                                           "protocol filtering not compatible"
2178                                           " with UDP layer");
2179         if (!(item_flags & l3m))
2180                 return rte_flow_error_set(error, EINVAL,
2181                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2182                                           "L3 is mandatory to filter on L4");
2183         if (item_flags & l4m)
2184                 return rte_flow_error_set(error, EINVAL,
2185                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2186                                           "multiple L4 layers not supported");
2187         if (!mask)
2188                 mask = &rte_flow_item_udp_mask;
2189         ret = mlx5_flow_item_acceptable
2190                 (item, (const uint8_t *)mask,
2191                  (const uint8_t *)&rte_flow_item_udp_mask,
2192                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2193                  error);
2194         if (ret < 0)
2195                 return ret;
2196         return 0;
2197 }
2198
2199 /**
2200  * Validate TCP item.
2201  *
2202  * @param[in] item
2203  *   Item specification.
2204  * @param[in] item_flags
2205  *   Bit-fields that holds the items detected until now.
2206  * @param[in] target_protocol
2207  *   The next protocol in the previous item.
2208  * @param[out] error
2209  *   Pointer to error structure.
2210  *
2211  * @return
2212  *   0 on success, a negative errno value otherwise and rte_errno is set.
2213  */
2214 int
2215 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2216                             uint64_t item_flags,
2217                             uint8_t target_protocol,
2218                             const struct rte_flow_item_tcp *flow_mask,
2219                             struct rte_flow_error *error)
2220 {
2221         const struct rte_flow_item_tcp *mask = item->mask;
2222         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2223         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2224                                       MLX5_FLOW_LAYER_OUTER_L3;
2225         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2226                                       MLX5_FLOW_LAYER_OUTER_L4;
2227         int ret;
2228
2229         MLX5_ASSERT(flow_mask);
2230         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2231                 return rte_flow_error_set(error, EINVAL,
2232                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2233                                           "protocol filtering not compatible"
2234                                           " with TCP layer");
2235         if (!(item_flags & l3m))
2236                 return rte_flow_error_set(error, EINVAL,
2237                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2238                                           "L3 is mandatory to filter on L4");
2239         if (item_flags & l4m)
2240                 return rte_flow_error_set(error, EINVAL,
2241                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2242                                           "multiple L4 layers not supported");
2243         if (!mask)
2244                 mask = &rte_flow_item_tcp_mask;
2245         ret = mlx5_flow_item_acceptable
2246                 (item, (const uint8_t *)mask,
2247                  (const uint8_t *)flow_mask,
2248                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2249                  error);
2250         if (ret < 0)
2251                 return ret;
2252         return 0;
2253 }
2254
2255 /**
2256  * Validate VXLAN item.
2257  *
2258  * @param[in] item
2259  *   Item specification.
2260  * @param[in] item_flags
2261  *   Bit-fields that holds the items detected until now.
2262  * @param[in] target_protocol
2263  *   The next protocol in the previous item.
2264  * @param[out] error
2265  *   Pointer to error structure.
2266  *
2267  * @return
2268  *   0 on success, a negative errno value otherwise and rte_errno is set.
2269  */
2270 int
2271 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2272                               uint64_t item_flags,
2273                               struct rte_flow_error *error)
2274 {
2275         const struct rte_flow_item_vxlan *spec = item->spec;
2276         const struct rte_flow_item_vxlan *mask = item->mask;
2277         int ret;
2278         union vni {
2279                 uint32_t vlan_id;
2280                 uint8_t vni[4];
2281         } id = { .vlan_id = 0, };
2282
2283
2284         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2285                 return rte_flow_error_set(error, ENOTSUP,
2286                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2287                                           "multiple tunnel layers not"
2288                                           " supported");
2289         /*
2290          * Verify only UDPv4 is present as defined in
2291          * https://tools.ietf.org/html/rfc7348
2292          */
2293         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2294                 return rte_flow_error_set(error, EINVAL,
2295                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2296                                           "no outer UDP layer found");
2297         if (!mask)
2298                 mask = &rte_flow_item_vxlan_mask;
2299         ret = mlx5_flow_item_acceptable
2300                 (item, (const uint8_t *)mask,
2301                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2302                  sizeof(struct rte_flow_item_vxlan),
2303                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2304         if (ret < 0)
2305                 return ret;
2306         if (spec) {
2307                 memcpy(&id.vni[1], spec->vni, 3);
2308                 memcpy(&id.vni[1], mask->vni, 3);
2309         }
2310         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2311                 return rte_flow_error_set(error, ENOTSUP,
2312                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2313                                           "VXLAN tunnel must be fully defined");
2314         return 0;
2315 }
2316
2317 /**
2318  * Validate VXLAN_GPE item.
2319  *
2320  * @param[in] item
2321  *   Item specification.
2322  * @param[in] item_flags
2323  *   Bit-fields that holds the items detected until now.
2324  * @param[in] priv
2325  *   Pointer to the private data structure.
2326  * @param[in] target_protocol
2327  *   The next protocol in the previous item.
2328  * @param[out] error
2329  *   Pointer to error structure.
2330  *
2331  * @return
2332  *   0 on success, a negative errno value otherwise and rte_errno is set.
2333  */
2334 int
2335 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2336                                   uint64_t item_flags,
2337                                   struct rte_eth_dev *dev,
2338                                   struct rte_flow_error *error)
2339 {
2340         struct mlx5_priv *priv = dev->data->dev_private;
2341         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2342         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2343         int ret;
2344         union vni {
2345                 uint32_t vlan_id;
2346                 uint8_t vni[4];
2347         } id = { .vlan_id = 0, };
2348
2349         if (!priv->config.l3_vxlan_en)
2350                 return rte_flow_error_set(error, ENOTSUP,
2351                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2352                                           "L3 VXLAN is not enabled by device"
2353                                           " parameter and/or not configured in"
2354                                           " firmware");
2355         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2356                 return rte_flow_error_set(error, ENOTSUP,
2357                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2358                                           "multiple tunnel layers not"
2359                                           " supported");
2360         /*
2361          * Verify only UDPv4 is present as defined in
2362          * https://tools.ietf.org/html/rfc7348
2363          */
2364         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2365                 return rte_flow_error_set(error, EINVAL,
2366                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2367                                           "no outer UDP layer found");
2368         if (!mask)
2369                 mask = &rte_flow_item_vxlan_gpe_mask;
2370         ret = mlx5_flow_item_acceptable
2371                 (item, (const uint8_t *)mask,
2372                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2373                  sizeof(struct rte_flow_item_vxlan_gpe),
2374                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2375         if (ret < 0)
2376                 return ret;
2377         if (spec) {
2378                 if (spec->protocol)
2379                         return rte_flow_error_set(error, ENOTSUP,
2380                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2381                                                   item,
2382                                                   "VxLAN-GPE protocol"
2383                                                   " not supported");
2384                 memcpy(&id.vni[1], spec->vni, 3);
2385                 memcpy(&id.vni[1], mask->vni, 3);
2386         }
2387         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2388                 return rte_flow_error_set(error, ENOTSUP,
2389                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2390                                           "VXLAN-GPE tunnel must be fully"
2391                                           " defined");
2392         return 0;
2393 }
2394 /**
2395  * Validate GRE Key item.
2396  *
2397  * @param[in] item
2398  *   Item specification.
2399  * @param[in] item_flags
2400  *   Bit flags to mark detected items.
2401  * @param[in] gre_item
2402  *   Pointer to gre_item
2403  * @param[out] error
2404  *   Pointer to error structure.
2405  *
2406  * @return
2407  *   0 on success, a negative errno value otherwise and rte_errno is set.
2408  */
2409 int
2410 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2411                                 uint64_t item_flags,
2412                                 const struct rte_flow_item *gre_item,
2413                                 struct rte_flow_error *error)
2414 {
2415         const rte_be32_t *mask = item->mask;
2416         int ret = 0;
2417         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2418         const struct rte_flow_item_gre *gre_spec;
2419         const struct rte_flow_item_gre *gre_mask;
2420
2421         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2424                                           "Multiple GRE key not support");
2425         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2426                 return rte_flow_error_set(error, ENOTSUP,
2427                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2428                                           "No preceding GRE header");
2429         if (item_flags & MLX5_FLOW_LAYER_INNER)
2430                 return rte_flow_error_set(error, ENOTSUP,
2431                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2432                                           "GRE key following a wrong item");
2433         gre_mask = gre_item->mask;
2434         if (!gre_mask)
2435                 gre_mask = &rte_flow_item_gre_mask;
2436         gre_spec = gre_item->spec;
2437         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2438                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2439                 return rte_flow_error_set(error, EINVAL,
2440                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2441                                           "Key bit must be on");
2442
2443         if (!mask)
2444                 mask = &gre_key_default_mask;
2445         ret = mlx5_flow_item_acceptable
2446                 (item, (const uint8_t *)mask,
2447                  (const uint8_t *)&gre_key_default_mask,
2448                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2449         return ret;
2450 }
2451
2452 /**
2453  * Validate GRE item.
2454  *
2455  * @param[in] item
2456  *   Item specification.
2457  * @param[in] item_flags
2458  *   Bit flags to mark detected items.
2459  * @param[in] target_protocol
2460  *   The next protocol in the previous item.
2461  * @param[out] error
2462  *   Pointer to error structure.
2463  *
2464  * @return
2465  *   0 on success, a negative errno value otherwise and rte_errno is set.
2466  */
2467 int
2468 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2469                             uint64_t item_flags,
2470                             uint8_t target_protocol,
2471                             struct rte_flow_error *error)
2472 {
2473         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2474         const struct rte_flow_item_gre *mask = item->mask;
2475         int ret;
2476         const struct rte_flow_item_gre nic_mask = {
2477                 .c_rsvd0_ver = RTE_BE16(0xB000),
2478                 .protocol = RTE_BE16(UINT16_MAX),
2479         };
2480
2481         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2484                                           "protocol filtering not compatible"
2485                                           " with this GRE layer");
2486         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2487                 return rte_flow_error_set(error, ENOTSUP,
2488                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2489                                           "multiple tunnel layers not"
2490                                           " supported");
2491         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2492                 return rte_flow_error_set(error, ENOTSUP,
2493                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2494                                           "L3 Layer is missing");
2495         if (!mask)
2496                 mask = &rte_flow_item_gre_mask;
2497         ret = mlx5_flow_item_acceptable
2498                 (item, (const uint8_t *)mask,
2499                  (const uint8_t *)&nic_mask,
2500                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2501                  error);
2502         if (ret < 0)
2503                 return ret;
2504 #ifndef HAVE_MLX5DV_DR
2505 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2506         if (spec && (spec->protocol & mask->protocol))
2507                 return rte_flow_error_set(error, ENOTSUP,
2508                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2509                                           "without MPLS support the"
2510                                           " specification cannot be used for"
2511                                           " filtering");
2512 #endif
2513 #endif
2514         return 0;
2515 }
2516
2517 /**
2518  * Validate Geneve item.
2519  *
2520  * @param[in] item
2521  *   Item specification.
2522  * @param[in] itemFlags
2523  *   Bit-fields that holds the items detected until now.
2524  * @param[in] enPriv
2525  *   Pointer to the private data structure.
2526  * @param[out] error
2527  *   Pointer to error structure.
2528  *
2529  * @return
2530  *   0 on success, a negative errno value otherwise and rte_errno is set.
2531  */
2532
2533 int
2534 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2535                                uint64_t item_flags,
2536                                struct rte_eth_dev *dev,
2537                                struct rte_flow_error *error)
2538 {
2539         struct mlx5_priv *priv = dev->data->dev_private;
2540         const struct rte_flow_item_geneve *spec = item->spec;
2541         const struct rte_flow_item_geneve *mask = item->mask;
2542         int ret;
2543         uint16_t gbhdr;
2544         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2545                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2546         const struct rte_flow_item_geneve nic_mask = {
2547                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2548                 .vni = "\xff\xff\xff",
2549                 .protocol = RTE_BE16(UINT16_MAX),
2550         };
2551
2552         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2553                 return rte_flow_error_set(error, ENOTSUP,
2554                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2555                                           "L3 Geneve is not enabled by device"
2556                                           " parameter and/or not configured in"
2557                                           " firmware");
2558         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2559                 return rte_flow_error_set(error, ENOTSUP,
2560                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2561                                           "multiple tunnel layers not"
2562                                           " supported");
2563         /*
2564          * Verify only UDPv4 is present as defined in
2565          * https://tools.ietf.org/html/rfc7348
2566          */
2567         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2568                 return rte_flow_error_set(error, EINVAL,
2569                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2570                                           "no outer UDP layer found");
2571         if (!mask)
2572                 mask = &rte_flow_item_geneve_mask;
2573         ret = mlx5_flow_item_acceptable
2574                                   (item, (const uint8_t *)mask,
2575                                    (const uint8_t *)&nic_mask,
2576                                    sizeof(struct rte_flow_item_geneve),
2577                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2578         if (ret)
2579                 return ret;
2580         if (spec) {
2581                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2582                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2583                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2584                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2585                         return rte_flow_error_set(error, ENOTSUP,
2586                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2587                                                   item,
2588                                                   "Geneve protocol unsupported"
2589                                                   " fields are being used");
2590                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2591                         return rte_flow_error_set
2592                                         (error, ENOTSUP,
2593                                          RTE_FLOW_ERROR_TYPE_ITEM,
2594                                          item,
2595                                          "Unsupported Geneve options length");
2596         }
2597         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2598                 return rte_flow_error_set
2599                                     (error, ENOTSUP,
2600                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2601                                      "Geneve tunnel must be fully defined");
2602         return 0;
2603 }
2604
2605 /**
2606  * Validate MPLS item.
2607  *
2608  * @param[in] dev
2609  *   Pointer to the rte_eth_dev structure.
2610  * @param[in] item
2611  *   Item specification.
2612  * @param[in] item_flags
2613  *   Bit-fields that holds the items detected until now.
2614  * @param[in] prev_layer
2615  *   The protocol layer indicated in previous item.
2616  * @param[out] error
2617  *   Pointer to error structure.
2618  *
2619  * @return
2620  *   0 on success, a negative errno value otherwise and rte_errno is set.
2621  */
2622 int
2623 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2624                              const struct rte_flow_item *item __rte_unused,
2625                              uint64_t item_flags __rte_unused,
2626                              uint64_t prev_layer __rte_unused,
2627                              struct rte_flow_error *error)
2628 {
2629 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2630         const struct rte_flow_item_mpls *mask = item->mask;
2631         struct mlx5_priv *priv = dev->data->dev_private;
2632         int ret;
2633
2634         if (!priv->config.mpls_en)
2635                 return rte_flow_error_set(error, ENOTSUP,
2636                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2637                                           "MPLS not supported or"
2638                                           " disabled in firmware"
2639                                           " configuration.");
2640         /* MPLS over IP, UDP, GRE is allowed */
2641         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2642                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2643                             MLX5_FLOW_LAYER_GRE |
2644                             MLX5_FLOW_LAYER_GRE_KEY)))
2645                 return rte_flow_error_set(error, EINVAL,
2646                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2647                                           "protocol filtering not compatible"
2648                                           " with MPLS layer");
2649         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2650         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2651             !(item_flags & MLX5_FLOW_LAYER_GRE))
2652                 return rte_flow_error_set(error, ENOTSUP,
2653                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2654                                           "multiple tunnel layers not"
2655                                           " supported");
2656         if (!mask)
2657                 mask = &rte_flow_item_mpls_mask;
2658         ret = mlx5_flow_item_acceptable
2659                 (item, (const uint8_t *)mask,
2660                  (const uint8_t *)&rte_flow_item_mpls_mask,
2661                  sizeof(struct rte_flow_item_mpls),
2662                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2663         if (ret < 0)
2664                 return ret;
2665         return 0;
2666 #else
2667         return rte_flow_error_set(error, ENOTSUP,
2668                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2669                                   "MPLS is not supported by Verbs, please"
2670                                   " update.");
2671 #endif
2672 }
2673
2674 /**
2675  * Validate NVGRE item.
2676  *
2677  * @param[in] item
2678  *   Item specification.
2679  * @param[in] item_flags
2680  *   Bit flags to mark detected items.
2681  * @param[in] target_protocol
2682  *   The next protocol in the previous item.
2683  * @param[out] error
2684  *   Pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success, a negative errno value otherwise and rte_errno is set.
2688  */
2689 int
2690 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2691                               uint64_t item_flags,
2692                               uint8_t target_protocol,
2693                               struct rte_flow_error *error)
2694 {
2695         const struct rte_flow_item_nvgre *mask = item->mask;
2696         int ret;
2697
2698         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2699                 return rte_flow_error_set(error, EINVAL,
2700                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2701                                           "protocol filtering not compatible"
2702                                           " with this GRE layer");
2703         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2704                 return rte_flow_error_set(error, ENOTSUP,
2705                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2706                                           "multiple tunnel layers not"
2707                                           " supported");
2708         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2709                 return rte_flow_error_set(error, ENOTSUP,
2710                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2711                                           "L3 Layer is missing");
2712         if (!mask)
2713                 mask = &rte_flow_item_nvgre_mask;
2714         ret = mlx5_flow_item_acceptable
2715                 (item, (const uint8_t *)mask,
2716                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2717                  sizeof(struct rte_flow_item_nvgre),
2718                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2719         if (ret < 0)
2720                 return ret;
2721         return 0;
2722 }
2723
2724 /**
2725  * Validate eCPRI item.
2726  *
2727  * @param[in] item
2728  *   Item specification.
2729  * @param[in] item_flags
2730  *   Bit-fields that holds the items detected until now.
2731  * @param[in] last_item
2732  *   Previous validated item in the pattern items.
2733  * @param[in] ether_type
2734  *   Type in the ethernet layer header (including dot1q).
2735  * @param[in] acc_mask
2736  *   Acceptable mask, if NULL default internal default mask
2737  *   will be used to check whether item fields are supported.
2738  * @param[out] error
2739  *   Pointer to error structure.
2740  *
2741  * @return
2742  *   0 on success, a negative errno value otherwise and rte_errno is set.
2743  */
2744 int
2745 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2746                               uint64_t item_flags,
2747                               uint64_t last_item,
2748                               uint16_t ether_type,
2749                               const struct rte_flow_item_ecpri *acc_mask,
2750                               struct rte_flow_error *error)
2751 {
2752         const struct rte_flow_item_ecpri *mask = item->mask;
2753         const struct rte_flow_item_ecpri nic_mask = {
2754                 .hdr = {
2755                         .common = {
2756                                 .u32 =
2757                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2758                                         .type = 0xFF,
2759                                         }).u32),
2760                         },
2761                         .dummy[0] = 0xFFFFFFFF,
2762                 },
2763         };
2764         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2765                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2766         struct rte_flow_item_ecpri mask_lo;
2767
2768         if (!(last_item & outer_l2_vlan) &&
2769             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
2770                 return rte_flow_error_set(error, EINVAL,
2771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
2773         if ((last_item & outer_l2_vlan) && ether_type &&
2774             ether_type != RTE_ETHER_TYPE_ECPRI)
2775                 return rte_flow_error_set(error, EINVAL,
2776                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2777                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
2778         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2779                 return rte_flow_error_set(error, EINVAL,
2780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2781                                           "eCPRI with tunnel is not supported right now");
2782         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2783                 return rte_flow_error_set(error, ENOTSUP,
2784                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2785                                           "multiple L3 layers not supported");
2786         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2787                 return rte_flow_error_set(error, EINVAL,
2788                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2789                                           "eCPRI cannot coexist with a TCP layer");
2790         /* In specification, eCPRI could be over UDP layer. */
2791         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2792                 return rte_flow_error_set(error, EINVAL,
2793                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2794                                           "eCPRI over UDP layer is not yet supported right now");
2795         /* Mask for type field in common header could be zero. */
2796         if (!mask)
2797                 mask = &rte_flow_item_ecpri_mask;
2798         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2799         /* Input mask is in big-endian format. */
2800         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2801                 return rte_flow_error_set(error, EINVAL,
2802                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2803                                           "partial mask is not supported for protocol");
2804         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2805                 return rte_flow_error_set(error, EINVAL,
2806                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2807                                           "message header mask must be after a type mask");
2808         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2809                                          acc_mask ? (const uint8_t *)acc_mask
2810                                                   : (const uint8_t *)&nic_mask,
2811                                          sizeof(struct rte_flow_item_ecpri),
2812                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2813 }
2814
2815 /**
2816  * Release resource related QUEUE/RSS action split.
2817  *
2818  * @param dev
2819  *   Pointer to Ethernet device.
2820  * @param flow
2821  *   Flow to release id's from.
2822  */
2823 static void
2824 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2825                              struct rte_flow *flow)
2826 {
2827         struct mlx5_priv *priv = dev->data->dev_private;
2828         uint32_t handle_idx;
2829         struct mlx5_flow_handle *dev_handle;
2830
2831         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2832                        handle_idx, dev_handle, next)
2833                 if (dev_handle->split_flow_id)
2834                         mlx5_ipool_free(priv->sh->ipool
2835                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
2836                                         dev_handle->split_flow_id);
2837 }
2838
2839 static int
2840 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2841                    const struct rte_flow_attr *attr __rte_unused,
2842                    const struct rte_flow_item items[] __rte_unused,
2843                    const struct rte_flow_action actions[] __rte_unused,
2844                    bool external __rte_unused,
2845                    int hairpin __rte_unused,
2846                    struct rte_flow_error *error)
2847 {
2848         return rte_flow_error_set(error, ENOTSUP,
2849                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2850 }
2851
2852 static struct mlx5_flow *
2853 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2854                   const struct rte_flow_attr *attr __rte_unused,
2855                   const struct rte_flow_item items[] __rte_unused,
2856                   const struct rte_flow_action actions[] __rte_unused,
2857                   struct rte_flow_error *error)
2858 {
2859         rte_flow_error_set(error, ENOTSUP,
2860                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2861         return NULL;
2862 }
2863
2864 static int
2865 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2866                     struct mlx5_flow *dev_flow __rte_unused,
2867                     const struct rte_flow_attr *attr __rte_unused,
2868                     const struct rte_flow_item items[] __rte_unused,
2869                     const struct rte_flow_action actions[] __rte_unused,
2870                     struct rte_flow_error *error)
2871 {
2872         return rte_flow_error_set(error, ENOTSUP,
2873                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2874 }
2875
2876 static int
2877 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2878                 struct rte_flow *flow __rte_unused,
2879                 struct rte_flow_error *error)
2880 {
2881         return rte_flow_error_set(error, ENOTSUP,
2882                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2883 }
2884
2885 static void
2886 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2887                  struct rte_flow *flow __rte_unused)
2888 {
2889 }
2890
2891 static void
2892 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2893                   struct rte_flow *flow __rte_unused)
2894 {
2895 }
2896
2897 static int
2898 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2899                 struct rte_flow *flow __rte_unused,
2900                 const struct rte_flow_action *actions __rte_unused,
2901                 void *data __rte_unused,
2902                 struct rte_flow_error *error)
2903 {
2904         return rte_flow_error_set(error, ENOTSUP,
2905                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2906 }
2907
2908 static int
2909 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
2910                       uint32_t domains __rte_unused,
2911                       uint32_t flags __rte_unused)
2912 {
2913         return 0;
2914 }
2915
2916 /* Void driver to protect from null pointer reference. */
2917 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2918         .validate = flow_null_validate,
2919         .prepare = flow_null_prepare,
2920         .translate = flow_null_translate,
2921         .apply = flow_null_apply,
2922         .remove = flow_null_remove,
2923         .destroy = flow_null_destroy,
2924         .query = flow_null_query,
2925         .sync_domain = flow_null_sync_domain,
2926 };
2927
2928 /**
2929  * Select flow driver type according to flow attributes and device
2930  * configuration.
2931  *
2932  * @param[in] dev
2933  *   Pointer to the dev structure.
2934  * @param[in] attr
2935  *   Pointer to the flow attributes.
2936  *
2937  * @return
2938  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2939  */
2940 static enum mlx5_flow_drv_type
2941 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2942 {
2943         struct mlx5_priv *priv = dev->data->dev_private;
2944         /* The OS can determine first a specific flow type (DV, VERBS) */
2945         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2946
2947         if (type != MLX5_FLOW_TYPE_MAX)
2948                 return type;
2949         /* If no OS specific type - continue with DV/VERBS selection */
2950         if (attr->transfer && priv->config.dv_esw_en)
2951                 type = MLX5_FLOW_TYPE_DV;
2952         if (!attr->transfer)
2953                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2954                                                  MLX5_FLOW_TYPE_VERBS;
2955         return type;
2956 }
2957
2958 #define flow_get_drv_ops(type) flow_drv_ops[type]
2959
2960 /**
2961  * Flow driver validation API. This abstracts calling driver specific functions.
2962  * The type of flow driver is determined according to flow attributes.
2963  *
2964  * @param[in] dev
2965  *   Pointer to the dev structure.
2966  * @param[in] attr
2967  *   Pointer to the flow attributes.
2968  * @param[in] items
2969  *   Pointer to the list of items.
2970  * @param[in] actions
2971  *   Pointer to the list of actions.
2972  * @param[in] external
2973  *   This flow rule is created by request external to PMD.
2974  * @param[in] hairpin
2975  *   Number of hairpin TX actions, 0 means classic flow.
2976  * @param[out] error
2977  *   Pointer to the error structure.
2978  *
2979  * @return
2980  *   0 on success, a negative errno value otherwise and rte_errno is set.
2981  */
2982 static inline int
2983 flow_drv_validate(struct rte_eth_dev *dev,
2984                   const struct rte_flow_attr *attr,
2985                   const struct rte_flow_item items[],
2986                   const struct rte_flow_action actions[],
2987                   bool external, int hairpin, struct rte_flow_error *error)
2988 {
2989         const struct mlx5_flow_driver_ops *fops;
2990         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2991
2992         fops = flow_get_drv_ops(type);
2993         return fops->validate(dev, attr, items, actions, external,
2994                               hairpin, error);
2995 }
2996
2997 /**
2998  * Flow driver preparation API. This abstracts calling driver specific
2999  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3000  * calculates the size of memory required for device flow, allocates the memory,
3001  * initializes the device flow and returns the pointer.
3002  *
3003  * @note
3004  *   This function initializes device flow structure such as dv or verbs in
3005  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3006  *   rest. For example, adding returning device flow to flow->dev_flow list and
3007  *   setting backward reference to the flow should be done out of this function.
3008  *   layers field is not filled either.
3009  *
3010  * @param[in] dev
3011  *   Pointer to the dev structure.
3012  * @param[in] attr
3013  *   Pointer to the flow attributes.
3014  * @param[in] items
3015  *   Pointer to the list of items.
3016  * @param[in] actions
3017  *   Pointer to the list of actions.
3018  * @param[in] flow_idx
3019  *   This memory pool index to the flow.
3020  * @param[out] error
3021  *   Pointer to the error structure.
3022  *
3023  * @return
3024  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3025  */
3026 static inline struct mlx5_flow *
3027 flow_drv_prepare(struct rte_eth_dev *dev,
3028                  const struct rte_flow *flow,
3029                  const struct rte_flow_attr *attr,
3030                  const struct rte_flow_item items[],
3031                  const struct rte_flow_action actions[],
3032                  uint32_t flow_idx,
3033                  struct rte_flow_error *error)
3034 {
3035         const struct mlx5_flow_driver_ops *fops;
3036         enum mlx5_flow_drv_type type = flow->drv_type;
3037         struct mlx5_flow *mlx5_flow = NULL;
3038
3039         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3040         fops = flow_get_drv_ops(type);
3041         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3042         if (mlx5_flow)
3043                 mlx5_flow->flow_idx = flow_idx;
3044         return mlx5_flow;
3045 }
3046
3047 /**
3048  * Flow driver translation API. This abstracts calling driver specific
3049  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3050  * translates a generic flow into a driver flow. flow_drv_prepare() must
3051  * precede.
3052  *
3053  * @note
3054  *   dev_flow->layers could be filled as a result of parsing during translation
3055  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3056  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3057  *   flow->actions could be overwritten even though all the expanded dev_flows
3058  *   have the same actions.
3059  *
3060  * @param[in] dev
3061  *   Pointer to the rte dev structure.
3062  * @param[in, out] dev_flow
3063  *   Pointer to the mlx5 flow.
3064  * @param[in] attr
3065  *   Pointer to the flow attributes.
3066  * @param[in] items
3067  *   Pointer to the list of items.
3068  * @param[in] actions
3069  *   Pointer to the list of actions.
3070  * @param[out] error
3071  *   Pointer to the error structure.
3072  *
3073  * @return
3074  *   0 on success, a negative errno value otherwise and rte_errno is set.
3075  */
3076 static inline int
3077 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3078                    const struct rte_flow_attr *attr,
3079                    const struct rte_flow_item items[],
3080                    const struct rte_flow_action actions[],
3081                    struct rte_flow_error *error)
3082 {
3083         const struct mlx5_flow_driver_ops *fops;
3084         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3085
3086         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3087         fops = flow_get_drv_ops(type);
3088         return fops->translate(dev, dev_flow, attr, items, actions, error);
3089 }
3090
3091 /**
3092  * Flow driver apply API. This abstracts calling driver specific functions.
3093  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3094  * translated driver flows on to device. flow_drv_translate() must precede.
3095  *
3096  * @param[in] dev
3097  *   Pointer to Ethernet device structure.
3098  * @param[in, out] flow
3099  *   Pointer to flow structure.
3100  * @param[out] error
3101  *   Pointer to error structure.
3102  *
3103  * @return
3104  *   0 on success, a negative errno value otherwise and rte_errno is set.
3105  */
3106 static inline int
3107 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3108                struct rte_flow_error *error)
3109 {
3110         const struct mlx5_flow_driver_ops *fops;
3111         enum mlx5_flow_drv_type type = flow->drv_type;
3112
3113         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3114         fops = flow_get_drv_ops(type);
3115         return fops->apply(dev, flow, error);
3116 }
3117
3118 /**
3119  * Flow driver destroy API. This abstracts calling driver specific functions.
3120  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3121  * on device and releases resources of the flow.
3122  *
3123  * @param[in] dev
3124  *   Pointer to Ethernet device.
3125  * @param[in, out] flow
3126  *   Pointer to flow structure.
3127  */
3128 static inline void
3129 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3130 {
3131         const struct mlx5_flow_driver_ops *fops;
3132         enum mlx5_flow_drv_type type = flow->drv_type;
3133
3134         flow_mreg_split_qrss_release(dev, flow);
3135         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3136         fops = flow_get_drv_ops(type);
3137         fops->destroy(dev, flow);
3138 }
3139
3140 /**
3141  * Get RSS action from the action list.
3142  *
3143  * @param[in] actions
3144  *   Pointer to the list of actions.
3145  *
3146  * @return
3147  *   Pointer to the RSS action if exist, else return NULL.
3148  */
3149 static const struct rte_flow_action_rss*
3150 flow_get_rss_action(const struct rte_flow_action actions[])
3151 {
3152         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3153                 switch (actions->type) {
3154                 case RTE_FLOW_ACTION_TYPE_RSS:
3155                         return (const struct rte_flow_action_rss *)
3156                                actions->conf;
3157                 default:
3158                         break;
3159                 }
3160         }
3161         return NULL;
3162 }
3163
3164 /**
3165  * Get ASO age action by index.
3166  *
3167  * @param[in] dev
3168  *   Pointer to the Ethernet device structure.
3169  * @param[in] age_idx
3170  *   Index to the ASO age action.
3171  *
3172  * @return
3173  *   The specified ASO age action.
3174  */
3175 struct mlx5_aso_age_action*
3176 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3177 {
3178         uint16_t pool_idx = age_idx & UINT16_MAX;
3179         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3180         struct mlx5_priv *priv = dev->data->dev_private;
3181         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3182         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3183
3184         return &pool->actions[offset - 1];
3185 }
3186
3187 /* maps shared action to translated non shared in some actions array */
3188 struct mlx5_translated_shared_action {
3189         struct rte_flow_shared_action *action; /**< Shared action */
3190         int index; /**< Index in related array of rte_flow_action */
3191 };
3192
3193 /**
3194  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3195  * non shared action if translation possible.
3196  * This functionality used to run same execution path for both shared & non
3197  * shared actions on flow create. All necessary preparations for shared
3198  * action handling should be preformed on *shared* actions list returned
3199  * from this call.
3200  *
3201  * @param[in] dev
3202  *   Pointer to Ethernet device.
3203  * @param[in] actions
3204  *   List of actions to translate.
3205  * @param[out] shared
3206  *   List to store translated shared actions.
3207  * @param[in, out] shared_n
3208  *   Size of *shared* array. On return should be updated with number of shared
3209  *   actions retrieved from the *actions* list.
3210  * @param[out] translated_actions
3211  *   List of actions where all shared actions were translated to non shared
3212  *   if possible. NULL if no translation took place.
3213  * @param[out] error
3214  *   Pointer to the error structure.
3215  *
3216  * @return
3217  *   0 on success, a negative errno value otherwise and rte_errno is set.
3218  */
3219 static int
3220 flow_shared_actions_translate(struct rte_eth_dev *dev,
3221                               const struct rte_flow_action actions[],
3222                               struct mlx5_translated_shared_action *shared,
3223                               int *shared_n,
3224                               struct rte_flow_action **translated_actions,
3225                               struct rte_flow_error *error)
3226 {
3227         struct mlx5_priv *priv = dev->data->dev_private;
3228         struct rte_flow_action *translated = NULL;
3229         size_t actions_size;
3230         int n;
3231         int copied_n = 0;
3232         struct mlx5_translated_shared_action *shared_end = NULL;
3233
3234         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3235                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3236                         continue;
3237                 if (copied_n == *shared_n) {
3238                         return rte_flow_error_set
3239                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3240                                  NULL, "too many shared actions");
3241                 }
3242                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3243                            sizeof(actions[n].conf));
3244                 shared[copied_n].index = n;
3245                 copied_n++;
3246         }
3247         n++;
3248         *shared_n = copied_n;
3249         if (!copied_n)
3250                 return 0;
3251         actions_size = sizeof(struct rte_flow_action) * n;
3252         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3253         if (!translated) {
3254                 rte_errno = ENOMEM;
3255                 return -ENOMEM;
3256         }
3257         memcpy(translated, actions, actions_size);
3258         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3259                 struct mlx5_shared_action_rss *shared_rss;
3260                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3261                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3262                 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET)
3263                                                                            - 1);
3264
3265                 switch (type) {
3266                 case MLX5_SHARED_ACTION_TYPE_RSS:
3267                         shared_rss = mlx5_ipool_get
3268                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3269                         translated[shared->index].type =
3270                                 RTE_FLOW_ACTION_TYPE_RSS;
3271                         translated[shared->index].conf =
3272                                 &shared_rss->origin;
3273                         break;
3274                 case MLX5_SHARED_ACTION_TYPE_AGE:
3275                         if (priv->sh->flow_hit_aso_en) {
3276                                 translated[shared->index].type =
3277                                         (enum rte_flow_action_type)
3278                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3279                                 translated[shared->index].conf =
3280                                                          (void *)(uintptr_t)idx;
3281                                 break;
3282                         }
3283                         /* Fall-through */
3284                 default:
3285                         mlx5_free(translated);
3286                         return rte_flow_error_set
3287                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3288                                  NULL, "invalid shared action type");
3289                 }
3290         }
3291         *translated_actions = translated;
3292         return 0;
3293 }
3294
3295 /**
3296  * Get Shared RSS action from the action list.
3297  *
3298  * @param[in] dev
3299  *   Pointer to Ethernet device.
3300  * @param[in] shared
3301  *   Pointer to the list of actions.
3302  * @param[in] shared_n
3303  *   Actions list length.
3304  *
3305  * @return
3306  *   The MLX5 RSS action ID if exists, otherwise return 0.
3307  */
3308 static uint32_t
3309 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3310                            struct mlx5_translated_shared_action *shared,
3311                            int shared_n)
3312 {
3313         struct mlx5_translated_shared_action *shared_end;
3314         struct mlx5_priv *priv = dev->data->dev_private;
3315         struct mlx5_shared_action_rss *shared_rss;
3316
3317
3318         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3319                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3320                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3321                 uint32_t idx = act_idx &
3322                                    ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1);
3323                 switch (type) {
3324                 case MLX5_SHARED_ACTION_TYPE_RSS:
3325                         shared_rss = mlx5_ipool_get
3326                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3327                                                                            idx);
3328                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3329                                            __ATOMIC_RELAXED);
3330                         return idx;
3331                 default:
3332                         break;
3333                 }
3334         }
3335         return 0;
3336 }
3337
3338 static unsigned int
3339 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3340 {
3341         const struct rte_flow_item *item;
3342         unsigned int has_vlan = 0;
3343
3344         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3345                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3346                         has_vlan = 1;
3347                         break;
3348                 }
3349         }
3350         if (has_vlan)
3351                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3352                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3353         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3354                                MLX5_EXPANSION_ROOT_OUTER;
3355 }
3356
3357 /**
3358  *  Get layer flags from the prefix flow.
3359  *
3360  *  Some flows may be split to several subflows, the prefix subflow gets the
3361  *  match items and the suffix sub flow gets the actions.
3362  *  Some actions need the user defined match item flags to get the detail for
3363  *  the action.
3364  *  This function helps the suffix flow to get the item layer flags from prefix
3365  *  subflow.
3366  *
3367  * @param[in] dev_flow
3368  *   Pointer the created preifx subflow.
3369  *
3370  * @return
3371  *   The layers get from prefix subflow.
3372  */
3373 static inline uint64_t
3374 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3375 {
3376         uint64_t layers = 0;
3377
3378         /*
3379          * Layers bits could be localization, but usually the compiler will
3380          * help to do the optimization work for source code.
3381          * If no decap actions, use the layers directly.
3382          */
3383         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3384                 return dev_flow->handle->layers;
3385         /* Convert L3 layers with decap action. */
3386         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3387                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3388         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3389                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3390         /* Convert L4 layers with decap action.  */
3391         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3392                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3393         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3394                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3395         return layers;
3396 }
3397
3398 /**
3399  * Get metadata split action information.
3400  *
3401  * @param[in] actions
3402  *   Pointer to the list of actions.
3403  * @param[out] qrss
3404  *   Pointer to the return pointer.
3405  * @param[out] qrss_type
3406  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3407  *   if no QUEUE/RSS is found.
3408  * @param[out] encap_idx
3409  *   Pointer to the index of the encap action if exists, otherwise the last
3410  *   action index.
3411  *
3412  * @return
3413  *   Total number of actions.
3414  */
3415 static int
3416 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3417                                        const struct rte_flow_action **qrss,
3418                                        int *encap_idx)
3419 {
3420         const struct rte_flow_action_raw_encap *raw_encap;
3421         int actions_n = 0;
3422         int raw_decap_idx = -1;
3423
3424         *encap_idx = -1;
3425         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3426                 switch (actions->type) {
3427                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3428                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3429                         *encap_idx = actions_n;
3430                         break;
3431                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3432                         raw_decap_idx = actions_n;
3433                         break;
3434                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3435                         raw_encap = actions->conf;
3436                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3437                                 *encap_idx = raw_decap_idx != -1 ?
3438                                                       raw_decap_idx : actions_n;
3439                         break;
3440                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3441                 case RTE_FLOW_ACTION_TYPE_RSS:
3442                         *qrss = actions;
3443                         break;
3444                 default:
3445                         break;
3446                 }
3447                 actions_n++;
3448         }
3449         if (*encap_idx == -1)
3450                 *encap_idx = actions_n;
3451         /* Count RTE_FLOW_ACTION_TYPE_END. */
3452         return actions_n + 1;
3453 }
3454
3455 /**
3456  * Check meter action from the action list.
3457  *
3458  * @param[in] actions
3459  *   Pointer to the list of actions.
3460  * @param[out] mtr
3461  *   Pointer to the meter exist flag.
3462  *
3463  * @return
3464  *   Total number of actions.
3465  */
3466 static int
3467 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3468 {
3469         int actions_n = 0;
3470
3471         MLX5_ASSERT(mtr);
3472         *mtr = 0;
3473         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3474                 switch (actions->type) {
3475                 case RTE_FLOW_ACTION_TYPE_METER:
3476                         *mtr = 1;
3477                         break;
3478                 default:
3479                         break;
3480                 }
3481                 actions_n++;
3482         }
3483         /* Count RTE_FLOW_ACTION_TYPE_END. */
3484         return actions_n + 1;
3485 }
3486
3487 /**
3488  * Check if the flow should be split due to hairpin.
3489  * The reason for the split is that in current HW we can't
3490  * support encap and push-vlan on Rx, so if a flow contains
3491  * these actions we move it to Tx.
3492  *
3493  * @param dev
3494  *   Pointer to Ethernet device.
3495  * @param[in] attr
3496  *   Flow rule attributes.
3497  * @param[in] actions
3498  *   Associated actions (list terminated by the END action).
3499  *
3500  * @return
3501  *   > 0 the number of actions and the flow should be split,
3502  *   0 when no split required.
3503  */
3504 static int
3505 flow_check_hairpin_split(struct rte_eth_dev *dev,
3506                          const struct rte_flow_attr *attr,
3507                          const struct rte_flow_action actions[])
3508 {
3509         int queue_action = 0;
3510         int action_n = 0;
3511         int split = 0;
3512         const struct rte_flow_action_queue *queue;
3513         const struct rte_flow_action_rss *rss;
3514         const struct rte_flow_action_raw_encap *raw_encap;
3515         const struct rte_eth_hairpin_conf *conf;
3516
3517         if (!attr->ingress)
3518                 return 0;
3519         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3520                 switch (actions->type) {
3521                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3522                         queue = actions->conf;
3523                         if (queue == NULL)
3524                                 return 0;
3525                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3526                         if (conf != NULL && !!conf->tx_explicit)
3527                                 return 0;
3528                         queue_action = 1;
3529                         action_n++;
3530                         break;
3531                 case RTE_FLOW_ACTION_TYPE_RSS:
3532                         rss = actions->conf;
3533                         if (rss == NULL || rss->queue_num == 0)
3534                                 return 0;
3535                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3536                         if (conf != NULL && !!conf->tx_explicit)
3537                                 return 0;
3538                         queue_action = 1;
3539                         action_n++;
3540                         break;
3541                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3542                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3543                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3544                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3545                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3546                         split++;
3547                         action_n++;
3548                         break;
3549                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3550                         raw_encap = actions->conf;
3551                         if (raw_encap->size >
3552                             (sizeof(struct rte_flow_item_eth) +
3553                              sizeof(struct rte_flow_item_ipv4)))
3554                                 split++;
3555                         action_n++;
3556                         break;
3557                 default:
3558                         action_n++;
3559                         break;
3560                 }
3561         }
3562         if (split && queue_action)
3563                 return action_n;
3564         return 0;
3565 }
3566
3567 /* Declare flow create/destroy prototype in advance. */
3568 static uint32_t
3569 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3570                  const struct rte_flow_attr *attr,
3571                  const struct rte_flow_item items[],
3572                  const struct rte_flow_action actions[],
3573                  bool external, struct rte_flow_error *error);
3574
3575 static void
3576 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3577                   uint32_t flow_idx);
3578
3579 struct mlx5_hlist_entry *
3580 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3581                        void *cb_ctx)
3582 {
3583         struct rte_eth_dev *dev = list->ctx;
3584         struct mlx5_priv *priv = dev->data->dev_private;
3585         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3586         struct mlx5_flow_mreg_copy_resource *mcp_res;
3587         struct rte_flow_error *error = ctx->error;
3588         uint32_t idx = 0;
3589         int ret;
3590         uint32_t mark_id = key;
3591         struct rte_flow_attr attr = {
3592                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3593                 .ingress = 1,
3594         };
3595         struct mlx5_rte_flow_item_tag tag_spec = {
3596                 .data = mark_id,
3597         };
3598         struct rte_flow_item items[] = {
3599                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3600         };
3601         struct rte_flow_action_mark ftag = {
3602                 .id = mark_id,
3603         };
3604         struct mlx5_flow_action_copy_mreg cp_mreg = {
3605                 .dst = REG_B,
3606                 .src = REG_NON,
3607         };
3608         struct rte_flow_action_jump jump = {
3609                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3610         };
3611         struct rte_flow_action actions[] = {
3612                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3613         };
3614
3615         /* Fill the register fileds in the flow. */
3616         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3617         if (ret < 0)
3618                 return NULL;
3619         tag_spec.id = ret;
3620         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3621         if (ret < 0)
3622                 return NULL;
3623         cp_mreg.src = ret;
3624         /* Provide the full width of FLAG specific value. */
3625         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3626                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3627         /* Build a new flow. */
3628         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3629                 items[0] = (struct rte_flow_item){
3630                         .type = (enum rte_flow_item_type)
3631                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3632                         .spec = &tag_spec,
3633                 };
3634                 items[1] = (struct rte_flow_item){
3635                         .type = RTE_FLOW_ITEM_TYPE_END,
3636                 };
3637                 actions[0] = (struct rte_flow_action){
3638                         .type = (enum rte_flow_action_type)
3639                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3640                         .conf = &ftag,
3641                 };
3642                 actions[1] = (struct rte_flow_action){
3643                         .type = (enum rte_flow_action_type)
3644                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3645                         .conf = &cp_mreg,
3646                 };
3647                 actions[2] = (struct rte_flow_action){
3648                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3649                         .conf = &jump,
3650                 };
3651                 actions[3] = (struct rte_flow_action){
3652                         .type = RTE_FLOW_ACTION_TYPE_END,
3653                 };
3654         } else {
3655                 /* Default rule, wildcard match. */
3656                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3657                 items[0] = (struct rte_flow_item){
3658                         .type = RTE_FLOW_ITEM_TYPE_END,
3659                 };
3660                 actions[0] = (struct rte_flow_action){
3661                         .type = (enum rte_flow_action_type)
3662                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3663                         .conf = &cp_mreg,
3664                 };
3665                 actions[1] = (struct rte_flow_action){
3666                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3667                         .conf = &jump,
3668                 };
3669                 actions[2] = (struct rte_flow_action){
3670                         .type = RTE_FLOW_ACTION_TYPE_END,
3671                 };
3672         }
3673         /* Build a new entry. */
3674         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3675         if (!mcp_res) {
3676                 rte_errno = ENOMEM;
3677                 return NULL;
3678         }
3679         mcp_res->idx = idx;
3680         /*
3681          * The copy Flows are not included in any list. There
3682          * ones are referenced from other Flows and can not
3683          * be applied, removed, deleted in ardbitrary order
3684          * by list traversing.
3685          */
3686         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3687                                          actions, false, error);
3688         if (!mcp_res->rix_flow) {
3689                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
3690                 return NULL;
3691         }
3692         return &mcp_res->hlist_ent;
3693 }
3694
3695 /**
3696  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3697  *
3698  * As mark_id is unique, if there's already a registered flow for the mark_id,
3699  * return by increasing the reference counter of the resource. Otherwise, create
3700  * the resource (mcp_res) and flow.
3701  *
3702  * Flow looks like,
3703  *   - If ingress port is ANY and reg_c[1] is mark_id,
3704  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3705  *
3706  * For default flow (zero mark_id), flow is like,
3707  *   - If ingress port is ANY,
3708  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3709  *
3710  * @param dev
3711  *   Pointer to Ethernet device.
3712  * @param mark_id
3713  *   ID of MARK action, zero means default flow for META.
3714  * @param[out] error
3715  *   Perform verbose error reporting if not NULL.
3716  *
3717  * @return
3718  *   Associated resource on success, NULL otherwise and rte_errno is set.
3719  */
3720 static struct mlx5_flow_mreg_copy_resource *
3721 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3722                           struct rte_flow_error *error)
3723 {
3724         struct mlx5_priv *priv = dev->data->dev_private;
3725         struct mlx5_hlist_entry *entry;
3726         struct mlx5_flow_cb_ctx ctx = {
3727                 .dev = dev,
3728                 .error = error,
3729         };
3730
3731         /* Check if already registered. */
3732         MLX5_ASSERT(priv->mreg_cp_tbl);
3733         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
3734         if (!entry)
3735                 return NULL;
3736         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
3737                             hlist_ent);
3738 }
3739
3740 void
3741 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
3742 {
3743         struct mlx5_flow_mreg_copy_resource *mcp_res =
3744                 container_of(entry, typeof(*mcp_res), hlist_ent);
3745         struct rte_eth_dev *dev = list->ctx;
3746         struct mlx5_priv *priv = dev->data->dev_private;
3747
3748         MLX5_ASSERT(mcp_res->rix_flow);
3749         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3750         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3751 }
3752
3753 /**
3754  * Release flow in RX_CP_TBL.
3755  *
3756  * @param dev
3757  *   Pointer to Ethernet device.
3758  * @flow
3759  *   Parent flow for wich copying is provided.
3760  */
3761 static void
3762 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3763                           struct rte_flow *flow)
3764 {
3765         struct mlx5_flow_mreg_copy_resource *mcp_res;
3766         struct mlx5_priv *priv = dev->data->dev_private;
3767
3768         if (!flow->rix_mreg_copy)
3769                 return;
3770         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3771                                  flow->rix_mreg_copy);
3772         if (!mcp_res || !priv->mreg_cp_tbl)
3773                 return;
3774         MLX5_ASSERT(mcp_res->rix_flow);
3775         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3776         flow->rix_mreg_copy = 0;
3777 }
3778
3779 /**
3780  * Remove the default copy action from RX_CP_TBL.
3781  *
3782  * This functions is called in the mlx5_dev_start(). No thread safe
3783  * is guaranteed.
3784  *
3785  * @param dev
3786  *   Pointer to Ethernet device.
3787  */
3788 static void
3789 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3790 {
3791         struct mlx5_hlist_entry *entry;
3792         struct mlx5_priv *priv = dev->data->dev_private;
3793
3794         /* Check if default flow is registered. */
3795         if (!priv->mreg_cp_tbl)
3796                 return;
3797         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
3798                                   MLX5_DEFAULT_COPY_ID, NULL);
3799         if (!entry)
3800                 return;
3801         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
3802 }
3803
3804 /**
3805  * Add the default copy action in in RX_CP_TBL.
3806  *
3807  * This functions is called in the mlx5_dev_start(). No thread safe
3808  * is guaranteed.
3809  *
3810  * @param dev
3811  *   Pointer to Ethernet device.
3812  * @param[out] error
3813  *   Perform verbose error reporting if not NULL.
3814  *
3815  * @return
3816  *   0 for success, negative value otherwise and rte_errno is set.
3817  */
3818 static int
3819 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3820                                   struct rte_flow_error *error)
3821 {
3822         struct mlx5_priv *priv = dev->data->dev_private;
3823         struct mlx5_flow_mreg_copy_resource *mcp_res;
3824
3825         /* Check whether extensive metadata feature is engaged. */
3826         if (!priv->config.dv_flow_en ||
3827             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3828             !mlx5_flow_ext_mreg_supported(dev) ||
3829             !priv->sh->dv_regc0_mask)
3830                 return 0;
3831         /*
3832          * Add default mreg copy flow may be called multiple time, but
3833          * only be called once in stop. Avoid register it twice.
3834          */
3835         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
3836                 return 0;
3837         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3838         if (!mcp_res)
3839                 return -rte_errno;
3840         return 0;
3841 }
3842
3843 /**
3844  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3845  *
3846  * All the flow having Q/RSS action should be split by
3847  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3848  * performs the following,
3849  *   - CQE->flow_tag := reg_c[1] (MARK)
3850  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3851  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3852  * but there should be a flow per each MARK ID set by MARK action.
3853  *
3854  * For the aforementioned reason, if there's a MARK action in flow's action
3855  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3856  * the MARK ID to CQE's flow_tag like,
3857  *   - If reg_c[1] is mark_id,
3858  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3859  *
3860  * For SET_META action which stores value in reg_c[0], as the destination is
3861  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3862  * MARK ID means the default flow. The default flow looks like,
3863  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3864  *
3865  * @param dev
3866  *   Pointer to Ethernet device.
3867  * @param flow
3868  *   Pointer to flow structure.
3869  * @param[in] actions
3870  *   Pointer to the list of actions.
3871  * @param[out] error
3872  *   Perform verbose error reporting if not NULL.
3873  *
3874  * @return
3875  *   0 on success, negative value otherwise and rte_errno is set.
3876  */
3877 static int
3878 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3879                             struct rte_flow *flow,
3880                             const struct rte_flow_action *actions,
3881                             struct rte_flow_error *error)
3882 {
3883         struct mlx5_priv *priv = dev->data->dev_private;
3884         struct mlx5_dev_config *config = &priv->config;
3885         struct mlx5_flow_mreg_copy_resource *mcp_res;
3886         const struct rte_flow_action_mark *mark;
3887
3888         /* Check whether extensive metadata feature is engaged. */
3889         if (!config->dv_flow_en ||
3890             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3891             !mlx5_flow_ext_mreg_supported(dev) ||
3892             !priv->sh->dv_regc0_mask)
3893                 return 0;
3894         /* Find MARK action. */
3895         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3896                 switch (actions->type) {
3897                 case RTE_FLOW_ACTION_TYPE_FLAG:
3898                         mcp_res = flow_mreg_add_copy_action
3899                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3900                         if (!mcp_res)
3901                                 return -rte_errno;
3902                         flow->rix_mreg_copy = mcp_res->idx;
3903                         return 0;
3904                 case RTE_FLOW_ACTION_TYPE_MARK:
3905                         mark = (const struct rte_flow_action_mark *)
3906                                 actions->conf;
3907                         mcp_res =
3908                                 flow_mreg_add_copy_action(dev, mark->id, error);
3909                         if (!mcp_res)
3910                                 return -rte_errno;
3911                         flow->rix_mreg_copy = mcp_res->idx;
3912                         return 0;
3913                 default:
3914                         break;
3915                 }
3916         }
3917         return 0;
3918 }
3919
3920 #define MLX5_MAX_SPLIT_ACTIONS 24
3921 #define MLX5_MAX_SPLIT_ITEMS 24
3922
3923 /**
3924  * Split the hairpin flow.
3925  * Since HW can't support encap and push-vlan on Rx, we move these
3926  * actions to Tx.
3927  * If the count action is after the encap then we also
3928  * move the count action. in this case the count will also measure
3929  * the outer bytes.
3930  *
3931  * @param dev
3932  *   Pointer to Ethernet device.
3933  * @param[in] actions
3934  *   Associated actions (list terminated by the END action).
3935  * @param[out] actions_rx
3936  *   Rx flow actions.
3937  * @param[out] actions_tx
3938  *   Tx flow actions..
3939  * @param[out] pattern_tx
3940  *   The pattern items for the Tx flow.
3941  * @param[out] flow_id
3942  *   The flow ID connected to this flow.
3943  *
3944  * @return
3945  *   0 on success.
3946  */
3947 static int
3948 flow_hairpin_split(struct rte_eth_dev *dev,
3949                    const struct rte_flow_action actions[],
3950                    struct rte_flow_action actions_rx[],
3951                    struct rte_flow_action actions_tx[],
3952                    struct rte_flow_item pattern_tx[],
3953                    uint32_t flow_id)
3954 {
3955         const struct rte_flow_action_raw_encap *raw_encap;
3956         const struct rte_flow_action_raw_decap *raw_decap;
3957         struct mlx5_rte_flow_action_set_tag *set_tag;
3958         struct rte_flow_action *tag_action;
3959         struct mlx5_rte_flow_item_tag *tag_item;
3960         struct rte_flow_item *item;
3961         char *addr;
3962         int encap = 0;
3963
3964         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3965                 switch (actions->type) {
3966                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3967                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3968                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3969                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3970                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3971                         rte_memcpy(actions_tx, actions,
3972                                sizeof(struct rte_flow_action));
3973                         actions_tx++;
3974                         break;
3975                 case RTE_FLOW_ACTION_TYPE_COUNT:
3976                         if (encap) {
3977                                 rte_memcpy(actions_tx, actions,
3978                                            sizeof(struct rte_flow_action));
3979                                 actions_tx++;
3980                         } else {
3981                                 rte_memcpy(actions_rx, actions,
3982                                            sizeof(struct rte_flow_action));
3983                                 actions_rx++;
3984                         }
3985                         break;
3986                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3987                         raw_encap = actions->conf;
3988                         if (raw_encap->size >
3989                             (sizeof(struct rte_flow_item_eth) +
3990                              sizeof(struct rte_flow_item_ipv4))) {
3991                                 memcpy(actions_tx, actions,
3992                                        sizeof(struct rte_flow_action));
3993                                 actions_tx++;
3994                                 encap = 1;
3995                         } else {
3996                                 rte_memcpy(actions_rx, actions,
3997                                            sizeof(struct rte_flow_action));
3998                                 actions_rx++;
3999                         }
4000                         break;
4001                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4002                         raw_decap = actions->conf;
4003                         if (raw_decap->size <
4004                             (sizeof(struct rte_flow_item_eth) +
4005                              sizeof(struct rte_flow_item_ipv4))) {
4006                                 memcpy(actions_tx, actions,
4007                                        sizeof(struct rte_flow_action));
4008                                 actions_tx++;
4009                         } else {
4010                                 rte_memcpy(actions_rx, actions,
4011                                            sizeof(struct rte_flow_action));
4012                                 actions_rx++;
4013                         }
4014                         break;
4015                 default:
4016                         rte_memcpy(actions_rx, actions,
4017                                    sizeof(struct rte_flow_action));
4018                         actions_rx++;
4019                         break;
4020                 }
4021         }
4022         /* Add set meta action and end action for the Rx flow. */
4023         tag_action = actions_rx;
4024         tag_action->type = (enum rte_flow_action_type)
4025                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4026         actions_rx++;
4027         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4028         actions_rx++;
4029         set_tag = (void *)actions_rx;
4030         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4031         MLX5_ASSERT(set_tag->id > REG_NON);
4032         set_tag->data = flow_id;
4033         tag_action->conf = set_tag;
4034         /* Create Tx item list. */
4035         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4036         addr = (void *)&pattern_tx[2];
4037         item = pattern_tx;
4038         item->type = (enum rte_flow_item_type)
4039                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4040         tag_item = (void *)addr;
4041         tag_item->data = flow_id;
4042         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4043         MLX5_ASSERT(set_tag->id > REG_NON);
4044         item->spec = tag_item;
4045         addr += sizeof(struct mlx5_rte_flow_item_tag);
4046         tag_item = (void *)addr;
4047         tag_item->data = UINT32_MAX;
4048         tag_item->id = UINT16_MAX;
4049         item->mask = tag_item;
4050         item->last = NULL;
4051         item++;
4052         item->type = RTE_FLOW_ITEM_TYPE_END;
4053         return 0;
4054 }
4055
4056 /**
4057  * The last stage of splitting chain, just creates the subflow
4058  * without any modification.
4059  *
4060  * @param[in] dev
4061  *   Pointer to Ethernet device.
4062  * @param[in] flow
4063  *   Parent flow structure pointer.
4064  * @param[in, out] sub_flow
4065  *   Pointer to return the created subflow, may be NULL.
4066  * @param[in] attr
4067  *   Flow rule attributes.
4068  * @param[in] items
4069  *   Pattern specification (list terminated by the END pattern item).
4070  * @param[in] actions
4071  *   Associated actions (list terminated by the END action).
4072  * @param[in] flow_split_info
4073  *   Pointer to flow split info structure.
4074  * @param[out] error
4075  *   Perform verbose error reporting if not NULL.
4076  * @return
4077  *   0 on success, negative value otherwise
4078  */
4079 static int
4080 flow_create_split_inner(struct rte_eth_dev *dev,
4081                         struct rte_flow *flow,
4082                         struct mlx5_flow **sub_flow,
4083                         const struct rte_flow_attr *attr,
4084                         const struct rte_flow_item items[],
4085                         const struct rte_flow_action actions[],
4086                         struct mlx5_flow_split_info *flow_split_info,
4087                         struct rte_flow_error *error)
4088 {
4089         struct mlx5_flow *dev_flow;
4090
4091         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4092                                     flow_split_info->flow_idx, error);
4093         if (!dev_flow)
4094                 return -rte_errno;
4095         dev_flow->flow = flow;
4096         dev_flow->external = flow_split_info->external;
4097         dev_flow->skip_scale = flow_split_info->skip_scale;
4098         /* Subflow object was created, we must include one in the list. */
4099         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4100                       dev_flow->handle, next);
4101         /*
4102          * If dev_flow is as one of the suffix flow, some actions in suffix
4103          * flow may need some user defined item layer flags, and pass the
4104          * Metadate rxq mark flag to suffix flow as well.
4105          */
4106         if (flow_split_info->prefix_layers)
4107                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4108         if (flow_split_info->prefix_mark)
4109                 dev_flow->handle->mark = 1;
4110         if (sub_flow)
4111                 *sub_flow = dev_flow;
4112         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4113 }
4114
4115 /**
4116  * Split the meter flow.
4117  *
4118  * As meter flow will split to three sub flow, other than meter
4119  * action, the other actions make sense to only meter accepts
4120  * the packet. If it need to be dropped, no other additional
4121  * actions should be take.
4122  *
4123  * One kind of special action which decapsulates the L3 tunnel
4124  * header will be in the prefix sub flow, as not to take the
4125  * L3 tunnel header into account.
4126  *
4127  * @param dev
4128  *   Pointer to Ethernet device.
4129  * @param[in] items
4130  *   Pattern specification (list terminated by the END pattern item).
4131  * @param[out] sfx_items
4132  *   Suffix flow match items (list terminated by the END pattern item).
4133  * @param[in] actions
4134  *   Associated actions (list terminated by the END action).
4135  * @param[out] actions_sfx
4136  *   Suffix flow actions.
4137  * @param[out] actions_pre
4138  *   Prefix flow actions.
4139  * @param[out] pattern_sfx
4140  *   The pattern items for the suffix flow.
4141  * @param[out] tag_sfx
4142  *   Pointer to suffix flow tag.
4143  *
4144  * @return
4145  *   0 on success.
4146  */
4147 static int
4148 flow_meter_split_prep(struct rte_eth_dev *dev,
4149                  const struct rte_flow_item items[],
4150                  struct rte_flow_item sfx_items[],
4151                  const struct rte_flow_action actions[],
4152                  struct rte_flow_action actions_sfx[],
4153                  struct rte_flow_action actions_pre[])
4154 {
4155         struct mlx5_priv *priv = dev->data->dev_private;
4156         struct rte_flow_action *tag_action = NULL;
4157         struct rte_flow_item *tag_item;
4158         struct mlx5_rte_flow_action_set_tag *set_tag;
4159         struct rte_flow_error error;
4160         const struct rte_flow_action_raw_encap *raw_encap;
4161         const struct rte_flow_action_raw_decap *raw_decap;
4162         struct mlx5_rte_flow_item_tag *tag_spec;
4163         struct mlx5_rte_flow_item_tag *tag_mask;
4164         uint32_t tag_id = 0;
4165         bool copy_vlan = false;
4166
4167         /* Prepare the actions for prefix and suffix flow. */
4168         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4169                 struct rte_flow_action **action_cur = NULL;
4170
4171                 switch (actions->type) {
4172                 case RTE_FLOW_ACTION_TYPE_METER:
4173                         /* Add the extra tag action first. */
4174                         tag_action = actions_pre;
4175                         tag_action->type = (enum rte_flow_action_type)
4176                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4177                         actions_pre++;
4178                         action_cur = &actions_pre;
4179                         break;
4180                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4181                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4182                         action_cur = &actions_pre;
4183                         break;
4184                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4185                         raw_encap = actions->conf;
4186                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4187                                 action_cur = &actions_pre;
4188                         break;
4189                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4190                         raw_decap = actions->conf;
4191                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4192                                 action_cur = &actions_pre;
4193                         break;
4194                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4195                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4196                         copy_vlan = true;
4197                         break;
4198                 default:
4199                         break;
4200                 }
4201                 if (!action_cur)
4202                         action_cur = &actions_sfx;
4203                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4204                 (*action_cur)++;
4205         }
4206         /* Add end action to the actions. */
4207         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4208         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4209         actions_pre++;
4210         /* Set the tag. */
4211         set_tag = (void *)actions_pre;
4212         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4213         mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4214                           &tag_id);
4215         if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
4216                 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
4217                         dev->data->port_id);
4218                 mlx5_ipool_free(priv->sh->ipool
4219                                 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
4220                 return 0;
4221         } else if (!tag_id) {
4222                 return 0;
4223         }
4224         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4225         assert(tag_action);
4226         tag_action->conf = set_tag;
4227         /* Prepare the suffix subflow items. */
4228         tag_item = sfx_items++;
4229         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4230                 int item_type = items->type;
4231
4232                 switch (item_type) {
4233                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4234                         memcpy(sfx_items, items, sizeof(*sfx_items));
4235                         sfx_items++;
4236                         break;
4237                 case RTE_FLOW_ITEM_TYPE_VLAN:
4238                         if (copy_vlan) {
4239                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4240                                 /*
4241                                  * Convert to internal match item, it is used
4242                                  * for vlan push and set vid.
4243                                  */
4244                                 sfx_items->type = (enum rte_flow_item_type)
4245                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4246                                 sfx_items++;
4247                         }
4248                         break;
4249                 default:
4250                         break;
4251                 }
4252         }
4253         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4254         sfx_items++;
4255         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4256         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4257         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4258         tag_mask = tag_spec + 1;
4259         tag_mask->data = 0xffffff00;
4260         tag_item->type = (enum rte_flow_item_type)
4261                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4262         tag_item->spec = tag_spec;
4263         tag_item->last = NULL;
4264         tag_item->mask = tag_mask;
4265         return tag_id;
4266 }
4267
4268 /**
4269  * Split action list having QUEUE/RSS for metadata register copy.
4270  *
4271  * Once Q/RSS action is detected in user's action list, the flow action
4272  * should be split in order to copy metadata registers, which will happen in
4273  * RX_CP_TBL like,
4274  *   - CQE->flow_tag := reg_c[1] (MARK)
4275  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4276  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4277  * This is because the last action of each flow must be a terminal action
4278  * (QUEUE, RSS or DROP).
4279  *
4280  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4281  * stored and kept in the mlx5_flow structure per each sub_flow.
4282  *
4283  * The Q/RSS action is replaced with,
4284  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4285  * And the following JUMP action is added at the end,
4286  *   - JUMP, to RX_CP_TBL.
4287  *
4288  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4289  * flow_create_split_metadata() routine. The flow will look like,
4290  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4291  *
4292  * @param dev
4293  *   Pointer to Ethernet device.
4294  * @param[out] split_actions
4295  *   Pointer to store split actions to jump to CP_TBL.
4296  * @param[in] actions
4297  *   Pointer to the list of original flow actions.
4298  * @param[in] qrss
4299  *   Pointer to the Q/RSS action.
4300  * @param[in] actions_n
4301  *   Number of original actions.
4302  * @param[out] error
4303  *   Perform verbose error reporting if not NULL.
4304  *
4305  * @return
4306  *   non-zero unique flow_id on success, otherwise 0 and
4307  *   error/rte_error are set.
4308  */
4309 static uint32_t
4310 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4311                           struct rte_flow_action *split_actions,
4312                           const struct rte_flow_action *actions,
4313                           const struct rte_flow_action *qrss,
4314                           int actions_n, struct rte_flow_error *error)
4315 {
4316         struct mlx5_priv *priv = dev->data->dev_private;
4317         struct mlx5_rte_flow_action_set_tag *set_tag;
4318         struct rte_flow_action_jump *jump;
4319         const int qrss_idx = qrss - actions;
4320         uint32_t flow_id = 0;
4321         int ret = 0;
4322
4323         /*
4324          * Given actions will be split
4325          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4326          * - Add jump to mreg CP_TBL.
4327          * As a result, there will be one more action.
4328          */
4329         ++actions_n;
4330         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4331         set_tag = (void *)(split_actions + actions_n);
4332         /*
4333          * If tag action is not set to void(it means we are not the meter
4334          * suffix flow), add the tag action. Since meter suffix flow already
4335          * has the tag added.
4336          */
4337         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4338                 /*
4339                  * Allocate the new subflow ID. This one is unique within
4340                  * device and not shared with representors. Otherwise,
4341                  * we would have to resolve multi-thread access synch
4342                  * issue. Each flow on the shared device is appended
4343                  * with source vport identifier, so the resulting
4344                  * flows will be unique in the shared (by master and
4345                  * representors) domain even if they have coinciding
4346                  * IDs.
4347                  */
4348                 mlx5_ipool_malloc(priv->sh->ipool
4349                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4350                 if (!flow_id)
4351                         return rte_flow_error_set(error, ENOMEM,
4352                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4353                                                   NULL, "can't allocate id "
4354                                                   "for split Q/RSS subflow");
4355                 /* Internal SET_TAG action to set flow ID. */
4356                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4357                         .data = flow_id,
4358                 };
4359                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4360                 if (ret < 0)
4361                         return ret;
4362                 set_tag->id = ret;
4363                 /* Construct new actions array. */
4364                 /* Replace QUEUE/RSS action. */
4365                 split_actions[qrss_idx] = (struct rte_flow_action){
4366                         .type = (enum rte_flow_action_type)
4367                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4368                         .conf = set_tag,
4369                 };
4370         }
4371         /* JUMP action to jump to mreg copy table (CP_TBL). */
4372         jump = (void *)(set_tag + 1);
4373         *jump = (struct rte_flow_action_jump){
4374                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4375         };
4376         split_actions[actions_n - 2] = (struct rte_flow_action){
4377                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4378                 .conf = jump,
4379         };
4380         split_actions[actions_n - 1] = (struct rte_flow_action){
4381                 .type = RTE_FLOW_ACTION_TYPE_END,
4382         };
4383         return flow_id;
4384 }
4385
4386 /**
4387  * Extend the given action list for Tx metadata copy.
4388  *
4389  * Copy the given action list to the ext_actions and add flow metadata register
4390  * copy action in order to copy reg_a set by WQE to reg_c[0].
4391  *
4392  * @param[out] ext_actions
4393  *   Pointer to the extended action list.
4394  * @param[in] actions
4395  *   Pointer to the list of actions.
4396  * @param[in] actions_n
4397  *   Number of actions in the list.
4398  * @param[out] error
4399  *   Perform verbose error reporting if not NULL.
4400  * @param[in] encap_idx
4401  *   The encap action inndex.
4402  *
4403  * @return
4404  *   0 on success, negative value otherwise
4405  */
4406 static int
4407 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4408                        struct rte_flow_action *ext_actions,
4409                        const struct rte_flow_action *actions,
4410                        int actions_n, struct rte_flow_error *error,
4411                        int encap_idx)
4412 {
4413         struct mlx5_flow_action_copy_mreg *cp_mreg =
4414                 (struct mlx5_flow_action_copy_mreg *)
4415                         (ext_actions + actions_n + 1);
4416         int ret;
4417
4418         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4419         if (ret < 0)
4420                 return ret;
4421         cp_mreg->dst = ret;
4422         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4423         if (ret < 0)
4424                 return ret;
4425         cp_mreg->src = ret;
4426         if (encap_idx != 0)
4427                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4428         if (encap_idx == actions_n - 1) {
4429                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4430                         .type = (enum rte_flow_action_type)
4431                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4432                         .conf = cp_mreg,
4433                 };
4434                 ext_actions[actions_n] = (struct rte_flow_action){
4435                         .type = RTE_FLOW_ACTION_TYPE_END,
4436                 };
4437         } else {
4438                 ext_actions[encap_idx] = (struct rte_flow_action){
4439                         .type = (enum rte_flow_action_type)
4440                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4441                         .conf = cp_mreg,
4442                 };
4443                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4444                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4445         }
4446         return 0;
4447 }
4448
4449 /**
4450  * Check the match action from the action list.
4451  *
4452  * @param[in] actions
4453  *   Pointer to the list of actions.
4454  * @param[in] attr
4455  *   Flow rule attributes.
4456  * @param[in] action
4457  *   The action to be check if exist.
4458  * @param[out] match_action_pos
4459  *   Pointer to the position of the matched action if exists, otherwise is -1.
4460  * @param[out] qrss_action_pos
4461  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4462  *
4463  * @return
4464  *   > 0 the total number of actions.
4465  *   0 if not found match action in action list.
4466  */
4467 static int
4468 flow_check_match_action(const struct rte_flow_action actions[],
4469                         const struct rte_flow_attr *attr,
4470                         enum rte_flow_action_type action,
4471                         int *match_action_pos, int *qrss_action_pos)
4472 {
4473         const struct rte_flow_action_sample *sample;
4474         int actions_n = 0;
4475         int jump_flag = 0;
4476         uint32_t ratio = 0;
4477         int sub_type = 0;
4478         int flag = 0;
4479
4480         *match_action_pos = -1;
4481         *qrss_action_pos = -1;
4482         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4483                 if (actions->type == action) {
4484                         flag = 1;
4485                         *match_action_pos = actions_n;
4486                 }
4487                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4488                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4489                         *qrss_action_pos = actions_n;
4490                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4491                         jump_flag = 1;
4492                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4493                         sample = actions->conf;
4494                         ratio = sample->ratio;
4495                         sub_type = ((const struct rte_flow_action *)
4496                                         (sample->actions))->type;
4497                 }
4498                 actions_n++;
4499         }
4500         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4501                 if (ratio == 1) {
4502                         /* JUMP Action not support for Mirroring;
4503                          * Mirroring support multi-destination;
4504                          */
4505                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4506                                 flag = 0;
4507                 }
4508         }
4509         /* Count RTE_FLOW_ACTION_TYPE_END. */
4510         return flag ? actions_n + 1 : 0;
4511 }
4512
4513 #define SAMPLE_SUFFIX_ITEM 2
4514
4515 /**
4516  * Split the sample flow.
4517  *
4518  * As sample flow will split to two sub flow, sample flow with
4519  * sample action, the other actions will move to new suffix flow.
4520  *
4521  * Also add unique tag id with tag action in the sample flow,
4522  * the same tag id will be as match in the suffix flow.
4523  *
4524  * @param dev
4525  *   Pointer to Ethernet device.
4526  * @param[in] fdb_tx
4527  *   FDB egress flow flag.
4528  * @param[out] sfx_items
4529  *   Suffix flow match items (list terminated by the END pattern item).
4530  * @param[in] actions
4531  *   Associated actions (list terminated by the END action).
4532  * @param[out] actions_sfx
4533  *   Suffix flow actions.
4534  * @param[out] actions_pre
4535  *   Prefix flow actions.
4536  * @param[in] actions_n
4537  *  The total number of actions.
4538  * @param[in] sample_action_pos
4539  *   The sample action position.
4540  * @param[in] qrss_action_pos
4541  *   The Queue/RSS action position.
4542  * @param[out] error
4543  *   Perform verbose error reporting if not NULL.
4544  *
4545  * @return
4546  *   0 on success, or unique flow_id, a negative errno value
4547  *   otherwise and rte_errno is set.
4548  */
4549 static int
4550 flow_sample_split_prep(struct rte_eth_dev *dev,
4551                        uint32_t fdb_tx,
4552                        struct rte_flow_item sfx_items[],
4553                        const struct rte_flow_action actions[],
4554                        struct rte_flow_action actions_sfx[],
4555                        struct rte_flow_action actions_pre[],
4556                        int actions_n,
4557                        int sample_action_pos,
4558                        int qrss_action_pos,
4559                        struct rte_flow_error *error)
4560 {
4561         struct mlx5_priv *priv = dev->data->dev_private;
4562         struct mlx5_rte_flow_action_set_tag *set_tag;
4563         struct mlx5_rte_flow_item_tag *tag_spec;
4564         struct mlx5_rte_flow_item_tag *tag_mask;
4565         uint32_t tag_id = 0;
4566         int index;
4567         int ret;
4568
4569         if (sample_action_pos < 0)
4570                 return rte_flow_error_set(error, EINVAL,
4571                                           RTE_FLOW_ERROR_TYPE_ACTION,
4572                                           NULL, "invalid position of sample "
4573                                           "action in list");
4574         if (!fdb_tx) {
4575                 /* Prepare the prefix tag action. */
4576                 set_tag = (void *)(actions_pre + actions_n + 1);
4577                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4578                 if (ret < 0)
4579                         return ret;
4580                 set_tag->id = ret;
4581                 mlx5_ipool_malloc(priv->sh->ipool
4582                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
4583                 set_tag->data = tag_id;
4584                 /* Prepare the suffix subflow items. */
4585                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4586                 tag_spec->data = tag_id;
4587                 tag_spec->id = set_tag->id;
4588                 tag_mask = tag_spec + 1;
4589                 tag_mask->data = UINT32_MAX;
4590                 sfx_items[0] = (struct rte_flow_item){
4591                         .type = (enum rte_flow_item_type)
4592                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4593                         .spec = tag_spec,
4594                         .last = NULL,
4595                         .mask = tag_mask,
4596                 };
4597                 sfx_items[1] = (struct rte_flow_item){
4598                         .type = (enum rte_flow_item_type)
4599                                 RTE_FLOW_ITEM_TYPE_END,
4600                 };
4601         }
4602         /* Prepare the actions for prefix and suffix flow. */
4603         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4604                 index = qrss_action_pos;
4605                 /* Put the preceding the Queue/RSS action into prefix flow. */
4606                 if (index != 0)
4607                         memcpy(actions_pre, actions,
4608                                sizeof(struct rte_flow_action) * index);
4609                 /* Put others preceding the sample action into prefix flow. */
4610                 if (sample_action_pos > index + 1)
4611                         memcpy(actions_pre + index, actions + index + 1,
4612                                sizeof(struct rte_flow_action) *
4613                                (sample_action_pos - index - 1));
4614                 index = sample_action_pos - 1;
4615                 /* Put Queue/RSS action into Suffix flow. */
4616                 memcpy(actions_sfx, actions + qrss_action_pos,
4617                        sizeof(struct rte_flow_action));
4618                 actions_sfx++;
4619         } else {
4620                 index = sample_action_pos;
4621                 if (index != 0)
4622                         memcpy(actions_pre, actions,
4623                                sizeof(struct rte_flow_action) * index);
4624         }
4625         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4626         if (!fdb_tx) {
4627                 actions_pre[index++] =
4628                         (struct rte_flow_action){
4629                         .type = (enum rte_flow_action_type)
4630                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4631                         .conf = set_tag,
4632                 };
4633         }
4634         memcpy(actions_pre + index, actions + sample_action_pos,
4635                sizeof(struct rte_flow_action));
4636         index += 1;
4637         actions_pre[index] = (struct rte_flow_action){
4638                 .type = (enum rte_flow_action_type)
4639                         RTE_FLOW_ACTION_TYPE_END,
4640         };
4641         /* Put the actions after sample into Suffix flow. */
4642         memcpy(actions_sfx, actions + sample_action_pos + 1,
4643                sizeof(struct rte_flow_action) *
4644                (actions_n - sample_action_pos - 1));
4645         return tag_id;
4646 }
4647
4648 /**
4649  * The splitting for metadata feature.
4650  *
4651  * - Q/RSS action on NIC Rx should be split in order to pass by
4652  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4653  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4654  *
4655  * - All the actions on NIC Tx should have a mreg copy action to
4656  *   copy reg_a from WQE to reg_c[0].
4657  *
4658  * @param dev
4659  *   Pointer to Ethernet device.
4660  * @param[in] flow
4661  *   Parent flow structure pointer.
4662  * @param[in] attr
4663  *   Flow rule attributes.
4664  * @param[in] items
4665  *   Pattern specification (list terminated by the END pattern item).
4666  * @param[in] actions
4667  *   Associated actions (list terminated by the END action).
4668  * @param[in] flow_split_info
4669  *   Pointer to flow split info structure.
4670  * @param[out] error
4671  *   Perform verbose error reporting if not NULL.
4672  * @return
4673  *   0 on success, negative value otherwise
4674  */
4675 static int
4676 flow_create_split_metadata(struct rte_eth_dev *dev,
4677                            struct rte_flow *flow,
4678                            const struct rte_flow_attr *attr,
4679                            const struct rte_flow_item items[],
4680                            const struct rte_flow_action actions[],
4681                            struct mlx5_flow_split_info *flow_split_info,
4682                            struct rte_flow_error *error)
4683 {
4684         struct mlx5_priv *priv = dev->data->dev_private;
4685         struct mlx5_dev_config *config = &priv->config;
4686         const struct rte_flow_action *qrss = NULL;
4687         struct rte_flow_action *ext_actions = NULL;
4688         struct mlx5_flow *dev_flow = NULL;
4689         uint32_t qrss_id = 0;
4690         int mtr_sfx = 0;
4691         size_t act_size;
4692         int actions_n;
4693         int encap_idx;
4694         int ret;
4695
4696         /* Check whether extensive metadata feature is engaged. */
4697         if (!config->dv_flow_en ||
4698             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4699             !mlx5_flow_ext_mreg_supported(dev))
4700                 return flow_create_split_inner(dev, flow, NULL, attr, items,
4701                                                actions, flow_split_info, error);
4702         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4703                                                            &encap_idx);
4704         if (qrss) {
4705                 /* Exclude hairpin flows from splitting. */
4706                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4707                         const struct rte_flow_action_queue *queue;
4708
4709                         queue = qrss->conf;
4710                         if (mlx5_rxq_get_type(dev, queue->index) ==
4711                             MLX5_RXQ_TYPE_HAIRPIN)
4712                                 qrss = NULL;
4713                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4714                         const struct rte_flow_action_rss *rss;
4715
4716                         rss = qrss->conf;
4717                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4718                             MLX5_RXQ_TYPE_HAIRPIN)
4719                                 qrss = NULL;
4720                 }
4721         }
4722         if (qrss) {
4723                 /* Check if it is in meter suffix table. */
4724                 mtr_sfx = attr->group == (attr->transfer ?
4725                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4726                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4727                 /*
4728                  * Q/RSS action on NIC Rx should be split in order to pass by
4729                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4730                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4731                  */
4732                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4733                            sizeof(struct rte_flow_action_set_tag) +
4734                            sizeof(struct rte_flow_action_jump);
4735                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4736                                           SOCKET_ID_ANY);
4737                 if (!ext_actions)
4738                         return rte_flow_error_set(error, ENOMEM,
4739                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4740                                                   NULL, "no memory to split "
4741                                                   "metadata flow");
4742                 /*
4743                  * If we are the suffix flow of meter, tag already exist.
4744                  * Set the tag action to void.
4745                  */
4746                 if (mtr_sfx)
4747                         ext_actions[qrss - actions].type =
4748                                                 RTE_FLOW_ACTION_TYPE_VOID;
4749                 else
4750                         ext_actions[qrss - actions].type =
4751                                                 (enum rte_flow_action_type)
4752                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4753                 /*
4754                  * Create the new actions list with removed Q/RSS action
4755                  * and appended set tag and jump to register copy table
4756                  * (RX_CP_TBL). We should preallocate unique tag ID here
4757                  * in advance, because it is needed for set tag action.
4758                  */
4759                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4760                                                     qrss, actions_n, error);
4761                 if (!mtr_sfx && !qrss_id) {
4762                         ret = -rte_errno;
4763                         goto exit;
4764                 }
4765         } else if (attr->egress && !attr->transfer) {
4766                 /*
4767                  * All the actions on NIC Tx should have a metadata register
4768                  * copy action to copy reg_a from WQE to reg_c[meta]
4769                  */
4770                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4771                            sizeof(struct mlx5_flow_action_copy_mreg);
4772                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4773                                           SOCKET_ID_ANY);
4774                 if (!ext_actions)
4775                         return rte_flow_error_set(error, ENOMEM,
4776                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4777                                                   NULL, "no memory to split "
4778                                                   "metadata flow");
4779                 /* Create the action list appended with copy register. */
4780                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4781                                              actions_n, error, encap_idx);
4782                 if (ret < 0)
4783                         goto exit;
4784         }
4785         /* Add the unmodified original or prefix subflow. */
4786         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
4787                                       items, ext_actions ? ext_actions :
4788                                       actions, flow_split_info, error);
4789         if (ret < 0)
4790                 goto exit;
4791         MLX5_ASSERT(dev_flow);
4792         if (qrss) {
4793                 const struct rte_flow_attr q_attr = {
4794                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4795                         .ingress = 1,
4796                 };
4797                 /* Internal PMD action to set register. */
4798                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4799                         .data = qrss_id,
4800                         .id = REG_NON,
4801                 };
4802                 struct rte_flow_item q_items[] = {
4803                         {
4804                                 .type = (enum rte_flow_item_type)
4805                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4806                                 .spec = &q_tag_spec,
4807                                 .last = NULL,
4808                                 .mask = NULL,
4809                         },
4810                         {
4811                                 .type = RTE_FLOW_ITEM_TYPE_END,
4812                         },
4813                 };
4814                 struct rte_flow_action q_actions[] = {
4815                         {
4816                                 .type = qrss->type,
4817                                 .conf = qrss->conf,
4818                         },
4819                         {
4820                                 .type = RTE_FLOW_ACTION_TYPE_END,
4821                         },
4822                 };
4823                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4824
4825                 /*
4826                  * Configure the tag item only if there is no meter subflow.
4827                  * Since tag is already marked in the meter suffix subflow
4828                  * we can just use the meter suffix items as is.
4829                  */
4830                 if (qrss_id) {
4831                         /* Not meter subflow. */
4832                         MLX5_ASSERT(!mtr_sfx);
4833                         /*
4834                          * Put unique id in prefix flow due to it is destroyed
4835                          * after suffix flow and id will be freed after there
4836                          * is no actual flows with this id and identifier
4837                          * reallocation becomes possible (for example, for
4838                          * other flows in other threads).
4839                          */
4840                         dev_flow->handle->split_flow_id = qrss_id;
4841                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4842                                                    error);
4843                         if (ret < 0)
4844                                 goto exit;
4845                         q_tag_spec.id = ret;
4846                 }
4847                 dev_flow = NULL;
4848                 /* Add suffix subflow to execute Q/RSS. */
4849                 flow_split_info->prefix_layers = layers;
4850                 flow_split_info->prefix_mark = 0;
4851                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4852                                               &q_attr, mtr_sfx ? items :
4853                                               q_items, q_actions,
4854                                               flow_split_info, error);
4855                 if (ret < 0)
4856                         goto exit;
4857                 /* qrss ID should be freed if failed. */
4858                 qrss_id = 0;
4859                 MLX5_ASSERT(dev_flow);
4860         }
4861
4862 exit:
4863         /*
4864          * We do not destroy the partially created sub_flows in case of error.
4865          * These ones are included into parent flow list and will be destroyed
4866          * by flow_drv_destroy.
4867          */
4868         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4869                         qrss_id);
4870         mlx5_free(ext_actions);
4871         return ret;
4872 }
4873
4874 /**
4875  * The splitting for meter feature.
4876  *
4877  * - The meter flow will be split to two flows as prefix and
4878  *   suffix flow. The packets make sense only it pass the prefix
4879  *   meter action.
4880  *
4881  * - Reg_C_5 is used for the packet to match betweend prefix and
4882  *   suffix flow.
4883  *
4884  * @param dev
4885  *   Pointer to Ethernet device.
4886  * @param[in] flow
4887  *   Parent flow structure pointer.
4888  * @param[in] attr
4889  *   Flow rule attributes.
4890  * @param[in] items
4891  *   Pattern specification (list terminated by the END pattern item).
4892  * @param[in] actions
4893  *   Associated actions (list terminated by the END action).
4894  * @param[in] flow_split_info
4895  *   Pointer to flow split info structure.
4896  * @param[out] error
4897  *   Perform verbose error reporting if not NULL.
4898  * @return
4899  *   0 on success, negative value otherwise
4900  */
4901 static int
4902 flow_create_split_meter(struct rte_eth_dev *dev,
4903                         struct rte_flow *flow,
4904                         const struct rte_flow_attr *attr,
4905                         const struct rte_flow_item items[],
4906                         const struct rte_flow_action actions[],
4907                         struct mlx5_flow_split_info *flow_split_info,
4908                         struct rte_flow_error *error)
4909 {
4910         struct mlx5_priv *priv = dev->data->dev_private;
4911         struct rte_flow_action *sfx_actions = NULL;
4912         struct rte_flow_action *pre_actions = NULL;
4913         struct rte_flow_item *sfx_items = NULL;
4914         struct mlx5_flow *dev_flow = NULL;
4915         struct rte_flow_attr sfx_attr = *attr;
4916         uint32_t mtr = 0;
4917         uint32_t mtr_tag_id = 0;
4918         size_t act_size;
4919         size_t item_size;
4920         int actions_n = 0;
4921         int ret;
4922
4923         if (priv->mtr_en)
4924                 actions_n = flow_check_meter_action(actions, &mtr);
4925         if (mtr) {
4926                 /* The five prefix actions: meter, decap, encap, tag, end. */
4927                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4928                            sizeof(struct mlx5_rte_flow_action_set_tag);
4929                 /* tag, vlan, port id, end. */
4930 #define METER_SUFFIX_ITEM 4
4931                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4932                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4933                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4934                                           0, SOCKET_ID_ANY);
4935                 if (!sfx_actions)
4936                         return rte_flow_error_set(error, ENOMEM,
4937                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4938                                                   NULL, "no memory to split "
4939                                                   "meter flow");
4940                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4941                              act_size);
4942                 pre_actions = sfx_actions + actions_n;
4943                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4944                                                    actions, sfx_actions,
4945                                                    pre_actions);
4946                 if (!mtr_tag_id) {
4947                         ret = -rte_errno;
4948                         goto exit;
4949                 }
4950                 /* Add the prefix subflow. */
4951                 flow_split_info->prefix_mark = 0;
4952                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4953                                               attr, items, pre_actions,
4954                                               flow_split_info, error);
4955                 if (ret) {
4956                         ret = -rte_errno;
4957                         goto exit;
4958                 }
4959                 dev_flow->handle->split_flow_id = mtr_tag_id;
4960                 /* Setting the sfx group atrr. */
4961                 sfx_attr.group = sfx_attr.transfer ?
4962                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4963                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4964                 flow_split_info->prefix_layers =
4965                                 flow_get_prefix_layer_flags(dev_flow);
4966                 flow_split_info->prefix_mark = dev_flow->handle->mark;
4967         }
4968         /* Add the prefix subflow. */
4969         ret = flow_create_split_metadata(dev, flow,
4970                                          &sfx_attr, sfx_items ?
4971                                          sfx_items : items,
4972                                          sfx_actions ? sfx_actions : actions,
4973                                          flow_split_info, error);
4974 exit:
4975         if (sfx_actions)
4976                 mlx5_free(sfx_actions);
4977         return ret;
4978 }
4979
4980 /**
4981  * The splitting for sample feature.
4982  *
4983  * Once Sample action is detected in the action list, the flow actions should
4984  * be split into prefix sub flow and suffix sub flow.
4985  *
4986  * The original items remain in the prefix sub flow, all actions preceding the
4987  * sample action and the sample action itself will be copied to the prefix
4988  * sub flow, the actions following the sample action will be copied to the
4989  * suffix sub flow, Queue action always be located in the suffix sub flow.
4990  *
4991  * In order to make the packet from prefix sub flow matches with suffix sub
4992  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
4993  * flow uses tag item with the unique flow id.
4994  *
4995  * @param dev
4996  *   Pointer to Ethernet device.
4997  * @param[in] flow
4998  *   Parent flow structure pointer.
4999  * @param[in] attr
5000  *   Flow rule attributes.
5001  * @param[in] items
5002  *   Pattern specification (list terminated by the END pattern item).
5003  * @param[in] actions
5004  *   Associated actions (list terminated by the END action).
5005  * @param[in] flow_split_info
5006  *   Pointer to flow split info structure.
5007  * @param[out] error
5008  *   Perform verbose error reporting if not NULL.
5009  * @return
5010  *   0 on success, negative value otherwise
5011  */
5012 static int
5013 flow_create_split_sample(struct rte_eth_dev *dev,
5014                          struct rte_flow *flow,
5015                          const struct rte_flow_attr *attr,
5016                          const struct rte_flow_item items[],
5017                          const struct rte_flow_action actions[],
5018                          struct mlx5_flow_split_info *flow_split_info,
5019                          struct rte_flow_error *error)
5020 {
5021         struct mlx5_priv *priv = dev->data->dev_private;
5022         struct rte_flow_action *sfx_actions = NULL;
5023         struct rte_flow_action *pre_actions = NULL;
5024         struct rte_flow_item *sfx_items = NULL;
5025         struct mlx5_flow *dev_flow = NULL;
5026         struct rte_flow_attr sfx_attr = *attr;
5027 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5028         struct mlx5_flow_dv_sample_resource *sample_res;
5029         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5030         struct mlx5_flow_tbl_resource *sfx_tbl;
5031         union mlx5_flow_tbl_key sfx_table_key;
5032 #endif
5033         size_t act_size;
5034         size_t item_size;
5035         uint32_t fdb_tx = 0;
5036         int32_t tag_id = 0;
5037         int actions_n = 0;
5038         int sample_action_pos;
5039         int qrss_action_pos;
5040         int ret = 0;
5041
5042         if (priv->sampler_en)
5043                 actions_n = flow_check_match_action(actions, attr,
5044                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5045                                         &sample_action_pos, &qrss_action_pos);
5046         if (actions_n) {
5047                 /* The prefix actions must includes sample, tag, end. */
5048                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5049                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5050                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5051                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5052                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5053                                           item_size), 0, SOCKET_ID_ANY);
5054                 if (!sfx_actions)
5055                         return rte_flow_error_set(error, ENOMEM,
5056                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5057                                                   NULL, "no memory to split "
5058                                                   "sample flow");
5059                 /* The representor_id is -1 for uplink. */
5060                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5061                 if (!fdb_tx)
5062                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5063                                         + act_size);
5064                 pre_actions = sfx_actions + actions_n;
5065                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5066                                                 actions, sfx_actions,
5067                                                 pre_actions, actions_n,
5068                                                 sample_action_pos,
5069                                                 qrss_action_pos, error);
5070                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5071                         ret = -rte_errno;
5072                         goto exit;
5073                 }
5074                 /* Add the prefix subflow. */
5075                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5076                                               items, pre_actions,
5077                                               flow_split_info, error);
5078                 if (ret) {
5079                         ret = -rte_errno;
5080                         goto exit;
5081                 }
5082                 dev_flow->handle->split_flow_id = tag_id;
5083 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5084                 /* Set the sfx group attr. */
5085                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5086                                         dev_flow->dv.sample_res;
5087                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5088                                         sample_res->normal_path_tbl;
5089                 sfx_tbl_data = container_of(sfx_tbl,
5090                                         struct mlx5_flow_tbl_data_entry, tbl);
5091                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5092                 sfx_attr.group = sfx_attr.transfer ?
5093                                         (sfx_table_key.table_id - 1) :
5094                                          sfx_table_key.table_id;
5095                 flow_split_info->prefix_layers =
5096                                 flow_get_prefix_layer_flags(dev_flow);
5097                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5098                 /* Suffix group level already be scaled with factor, set
5099                  * skip_scale to 1 to avoid scale again in translation.
5100                  */
5101                 flow_split_info->skip_scale = 1;
5102 #endif
5103         }
5104         /* Add the suffix subflow. */
5105         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5106                                       sfx_items ? sfx_items : items,
5107                                       sfx_actions ? sfx_actions : actions,
5108                                       flow_split_info, error);
5109 exit:
5110         if (sfx_actions)
5111                 mlx5_free(sfx_actions);
5112         return ret;
5113 }
5114
5115 /**
5116  * Split the flow to subflow set. The splitters might be linked
5117  * in the chain, like this:
5118  * flow_create_split_outer() calls:
5119  *   flow_create_split_meter() calls:
5120  *     flow_create_split_metadata(meter_subflow_0) calls:
5121  *       flow_create_split_inner(metadata_subflow_0)
5122  *       flow_create_split_inner(metadata_subflow_1)
5123  *       flow_create_split_inner(metadata_subflow_2)
5124  *     flow_create_split_metadata(meter_subflow_1) calls:
5125  *       flow_create_split_inner(metadata_subflow_0)
5126  *       flow_create_split_inner(metadata_subflow_1)
5127  *       flow_create_split_inner(metadata_subflow_2)
5128  *
5129  * This provide flexible way to add new levels of flow splitting.
5130  * The all of successfully created subflows are included to the
5131  * parent flow dev_flow list.
5132  *
5133  * @param dev
5134  *   Pointer to Ethernet device.
5135  * @param[in] flow
5136  *   Parent flow structure pointer.
5137  * @param[in] attr
5138  *   Flow rule attributes.
5139  * @param[in] items
5140  *   Pattern specification (list terminated by the END pattern item).
5141  * @param[in] actions
5142  *   Associated actions (list terminated by the END action).
5143  * @param[in] flow_split_info
5144  *   Pointer to flow split info structure.
5145  * @param[out] error
5146  *   Perform verbose error reporting if not NULL.
5147  * @return
5148  *   0 on success, negative value otherwise
5149  */
5150 static int
5151 flow_create_split_outer(struct rte_eth_dev *dev,
5152                         struct rte_flow *flow,
5153                         const struct rte_flow_attr *attr,
5154                         const struct rte_flow_item items[],
5155                         const struct rte_flow_action actions[],
5156                         struct mlx5_flow_split_info *flow_split_info,
5157                         struct rte_flow_error *error)
5158 {
5159         int ret;
5160
5161         ret = flow_create_split_sample(dev, flow, attr, items,
5162                                        actions, flow_split_info, error);
5163         MLX5_ASSERT(ret <= 0);
5164         return ret;
5165 }
5166
5167 static struct mlx5_flow_tunnel *
5168 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5169                       const struct rte_flow_attr *attr,
5170                       const struct rte_flow_item items[],
5171                       const struct rte_flow_action actions[])
5172 {
5173         struct mlx5_flow_tunnel *tunnel;
5174
5175 #pragma GCC diagnostic push
5176 #pragma GCC diagnostic ignored "-Wcast-qual"
5177         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5178                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5179         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5180                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5181         else
5182                 tunnel = NULL;
5183 #pragma GCC diagnostic pop
5184
5185         return tunnel;
5186 }
5187
5188 /**
5189  * Adjust flow RSS workspace if needed.
5190  *
5191  * @param wks
5192  *   Pointer to thread flow work space.
5193  * @param rss_desc
5194  *   Pointer to RSS descriptor.
5195  * @param[in] nrssq_num
5196  *   New RSS queue number.
5197  *
5198  * @return
5199  *   0 on success, -1 otherwise and rte_errno is set.
5200  */
5201 static int
5202 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
5203                           struct mlx5_flow_rss_desc *rss_desc,
5204                           uint32_t nrssq_num)
5205 {
5206         if (likely(nrssq_num <= wks->rssq_num))
5207                 return 0;
5208         rss_desc->queue = realloc(rss_desc->queue,
5209                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
5210         if (!rss_desc->queue) {
5211                 rte_errno = ENOMEM;
5212                 return -1;
5213         }
5214         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
5215         return 0;
5216 }
5217
5218 /**
5219  * Create a flow and add it to @p list.
5220  *
5221  * @param dev
5222  *   Pointer to Ethernet device.
5223  * @param list
5224  *   Pointer to a TAILQ flow list. If this parameter NULL,
5225  *   no list insertion occurred, flow is just created,
5226  *   this is caller's responsibility to track the
5227  *   created flow.
5228  * @param[in] attr
5229  *   Flow rule attributes.
5230  * @param[in] items
5231  *   Pattern specification (list terminated by the END pattern item).
5232  * @param[in] actions
5233  *   Associated actions (list terminated by the END action).
5234  * @param[in] external
5235  *   This flow rule is created by request external to PMD.
5236  * @param[out] error
5237  *   Perform verbose error reporting if not NULL.
5238  *
5239  * @return
5240  *   A flow index on success, 0 otherwise and rte_errno is set.
5241  */
5242 static uint32_t
5243 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5244                  const struct rte_flow_attr *attr,
5245                  const struct rte_flow_item items[],
5246                  const struct rte_flow_action original_actions[],
5247                  bool external, struct rte_flow_error *error)
5248 {
5249         struct mlx5_priv *priv = dev->data->dev_private;
5250         struct rte_flow *flow = NULL;
5251         struct mlx5_flow *dev_flow;
5252         const struct rte_flow_action_rss *rss;
5253         struct mlx5_translated_shared_action
5254                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5255         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5256         union {
5257                 struct mlx5_flow_expand_rss buf;
5258                 uint8_t buffer[2048];
5259         } expand_buffer;
5260         union {
5261                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5262                 uint8_t buffer[2048];
5263         } actions_rx;
5264         union {
5265                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5266                 uint8_t buffer[2048];
5267         } actions_hairpin_tx;
5268         union {
5269                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5270                 uint8_t buffer[2048];
5271         } items_tx;
5272         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5273         struct mlx5_flow_rss_desc *rss_desc;
5274         const struct rte_flow_action *p_actions_rx;
5275         uint32_t i;
5276         uint32_t idx = 0;
5277         int hairpin_flow;
5278         struct rte_flow_attr attr_tx = { .priority = 0 };
5279         const struct rte_flow_action *actions;
5280         struct rte_flow_action *translated_actions = NULL;
5281         struct mlx5_flow_tunnel *tunnel;
5282         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5283         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
5284         struct mlx5_flow_split_info flow_split_info = {
5285                 .external = !!external,
5286                 .skip_scale = 0,
5287                 .flow_idx = 0,
5288                 .prefix_mark = 0,
5289                 .prefix_layers = 0
5290         };
5291         int ret;
5292
5293         MLX5_ASSERT(wks);
5294         rss_desc = &wks->rss_desc;
5295         ret = flow_shared_actions_translate(dev, original_actions,
5296                                             shared_actions,
5297                                             &shared_actions_n,
5298                                             &translated_actions, error);
5299         if (ret < 0) {
5300                 MLX5_ASSERT(translated_actions == NULL);
5301                 return 0;
5302         }
5303         actions = translated_actions ? translated_actions : original_actions;
5304         p_actions_rx = actions;
5305         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5306         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
5307                                 external, hairpin_flow, error);
5308         if (ret < 0)
5309                 goto error_before_hairpin_split;
5310         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5311         if (!flow) {
5312                 rte_errno = ENOMEM;
5313                 goto error_before_hairpin_split;
5314         }
5315         if (hairpin_flow > 0) {
5316                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5317                         rte_errno = EINVAL;
5318                         goto error_before_hairpin_split;
5319                 }
5320                 flow_hairpin_split(dev, actions, actions_rx.actions,
5321                                    actions_hairpin_tx.actions, items_tx.items,
5322                                    idx);
5323                 p_actions_rx = actions_rx.actions;
5324         }
5325         flow_split_info.flow_idx = idx;
5326         flow->drv_type = flow_get_drv_type(dev, attr);
5327         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5328                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5329         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
5330         rss = flow_get_rss_action(p_actions_rx);
5331         if (rss) {
5332                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
5333                         return 0;
5334                 /*
5335                  * The following information is required by
5336                  * mlx5_flow_hashfields_adjust() in advance.
5337                  */
5338                 rss_desc->level = rss->level;
5339                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5340                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5341         }
5342         flow->dev_handles = 0;
5343         if (rss && rss->types) {
5344                 unsigned int graph_root;
5345
5346                 graph_root = find_graph_root(items, rss->level);
5347                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5348                                            items, rss->types,
5349                                            mlx5_support_expansion, graph_root);
5350                 MLX5_ASSERT(ret > 0 &&
5351                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5352         } else {
5353                 buf->entries = 1;
5354                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5355         }
5356         rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions,
5357                                                       shared_actions_n);
5358         for (i = 0; i < buf->entries; ++i) {
5359                 /* Initialize flow split data. */
5360                 flow_split_info.prefix_layers = 0;
5361                 flow_split_info.prefix_mark = 0;
5362                 flow_split_info.skip_scale = 0;
5363                 /*
5364                  * The splitter may create multiple dev_flows,
5365                  * depending on configuration. In the simplest
5366                  * case it just creates unmodified original flow.
5367                  */
5368                 ret = flow_create_split_outer(dev, flow, attr,
5369                                               buf->entry[i].pattern,
5370                                               p_actions_rx, &flow_split_info,
5371                                               error);
5372                 if (ret < 0)
5373                         goto error;
5374                 if (is_flow_tunnel_steer_rule(dev, attr,
5375                                               buf->entry[i].pattern,
5376                                               p_actions_rx)) {
5377                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5378                                                            p_actions_rx,
5379                                                            idx,
5380                                                            &default_miss_ctx,
5381                                                            error);
5382                         if (ret < 0) {
5383                                 mlx5_free(default_miss_ctx.queue);
5384                                 goto error;
5385                         }
5386                 }
5387         }
5388         /* Create the tx flow. */
5389         if (hairpin_flow) {
5390                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5391                 attr_tx.ingress = 0;
5392                 attr_tx.egress = 1;
5393                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5394                                          actions_hairpin_tx.actions,
5395                                          idx, error);
5396                 if (!dev_flow)
5397                         goto error;
5398                 dev_flow->flow = flow;
5399                 dev_flow->external = 0;
5400                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5401                               dev_flow->handle, next);
5402                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5403                                          items_tx.items,
5404                                          actions_hairpin_tx.actions, error);
5405                 if (ret < 0)
5406                         goto error;
5407         }
5408         /*
5409          * Update the metadata register copy table. If extensive
5410          * metadata feature is enabled and registers are supported
5411          * we might create the extra rte_flow for each unique
5412          * MARK/FLAG action ID.
5413          *
5414          * The table is updated for ingress Flows only, because
5415          * the egress Flows belong to the different device and
5416          * copy table should be updated in peer NIC Rx domain.
5417          */
5418         if (attr->ingress &&
5419             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5420                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5421                 if (ret)
5422                         goto error;
5423         }
5424         /*
5425          * If the flow is external (from application) OR device is started,
5426          * OR mreg discover, then apply immediately.
5427          */
5428         if (external || dev->data->dev_started ||
5429             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
5430              attr->priority == MLX5_FLOW_PRIO_RSVD)) {
5431                 ret = flow_drv_apply(dev, flow, error);
5432                 if (ret < 0)
5433                         goto error;
5434         }
5435         if (list) {
5436                 rte_spinlock_lock(&priv->flow_list_lock);
5437                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5438                              flow, next);
5439                 rte_spinlock_unlock(&priv->flow_list_lock);
5440         }
5441         flow_rxq_flags_set(dev, flow);
5442         rte_free(translated_actions);
5443         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5444         if (tunnel) {
5445                 flow->tunnel = 1;
5446                 flow->tunnel_id = tunnel->tunnel_id;
5447                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5448                 mlx5_free(default_miss_ctx.queue);
5449         }
5450         mlx5_flow_pop_thread_workspace();
5451         return idx;
5452 error:
5453         MLX5_ASSERT(flow);
5454         ret = rte_errno; /* Save rte_errno before cleanup. */
5455         flow_mreg_del_copy_action(dev, flow);
5456         flow_drv_destroy(dev, flow);
5457         if (rss_desc->shared_rss)
5458                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
5459                         mlx5_ipool_get
5460                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
5461                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
5462         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5463         rte_errno = ret; /* Restore rte_errno. */
5464         ret = rte_errno;
5465         rte_errno = ret;
5466         mlx5_flow_pop_thread_workspace();
5467 error_before_hairpin_split:
5468         rte_free(translated_actions);
5469         return 0;
5470 }
5471
5472 /**
5473  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5474  * incoming packets to table 1.
5475  *
5476  * Other flow rules, requested for group n, will be created in
5477  * e-switch table n+1.
5478  * Jump action to e-switch group n will be created to group n+1.
5479  *
5480  * Used when working in switchdev mode, to utilise advantages of table 1
5481  * and above.
5482  *
5483  * @param dev
5484  *   Pointer to Ethernet device.
5485  *
5486  * @return
5487  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5488  */
5489 struct rte_flow *
5490 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5491 {
5492         const struct rte_flow_attr attr = {
5493                 .group = 0,
5494                 .priority = 0,
5495                 .ingress = 1,
5496                 .egress = 0,
5497                 .transfer = 1,
5498         };
5499         const struct rte_flow_item pattern = {
5500                 .type = RTE_FLOW_ITEM_TYPE_END,
5501         };
5502         struct rte_flow_action_jump jump = {
5503                 .group = 1,
5504         };
5505         const struct rte_flow_action actions[] = {
5506                 {
5507                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5508                         .conf = &jump,
5509                 },
5510                 {
5511                         .type = RTE_FLOW_ACTION_TYPE_END,
5512                 },
5513         };
5514         struct mlx5_priv *priv = dev->data->dev_private;
5515         struct rte_flow_error error;
5516
5517         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5518                                                    &attr, &pattern,
5519                                                    actions, false, &error);
5520 }
5521
5522 /**
5523  * Validate a flow supported by the NIC.
5524  *
5525  * @see rte_flow_validate()
5526  * @see rte_flow_ops
5527  */
5528 int
5529 mlx5_flow_validate(struct rte_eth_dev *dev,
5530                    const struct rte_flow_attr *attr,
5531                    const struct rte_flow_item items[],
5532                    const struct rte_flow_action original_actions[],
5533                    struct rte_flow_error *error)
5534 {
5535         int hairpin_flow;
5536         struct mlx5_translated_shared_action
5537                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5538         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5539         const struct rte_flow_action *actions;
5540         struct rte_flow_action *translated_actions = NULL;
5541         int ret = flow_shared_actions_translate(dev, original_actions,
5542                                                 shared_actions,
5543                                                 &shared_actions_n,
5544                                                 &translated_actions, error);
5545
5546         if (ret)
5547                 return ret;
5548         actions = translated_actions ? translated_actions : original_actions;
5549         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5550         ret = flow_drv_validate(dev, attr, items, actions,
5551                                 true, hairpin_flow, error);
5552         rte_free(translated_actions);
5553         return ret;
5554 }
5555
5556 /**
5557  * Create a flow.
5558  *
5559  * @see rte_flow_create()
5560  * @see rte_flow_ops
5561  */
5562 struct rte_flow *
5563 mlx5_flow_create(struct rte_eth_dev *dev,
5564                  const struct rte_flow_attr *attr,
5565                  const struct rte_flow_item items[],
5566                  const struct rte_flow_action actions[],
5567                  struct rte_flow_error *error)
5568 {
5569         struct mlx5_priv *priv = dev->data->dev_private;
5570
5571         /*
5572          * If the device is not started yet, it is not allowed to created a
5573          * flow from application. PMD default flows and traffic control flows
5574          * are not affected.
5575          */
5576         if (unlikely(!dev->data->dev_started)) {
5577                 DRV_LOG(DEBUG, "port %u is not started when "
5578                         "inserting a flow", dev->data->port_id);
5579                 rte_flow_error_set(error, ENODEV,
5580                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5581                                    NULL,
5582                                    "port not started");
5583                 return NULL;
5584         }
5585
5586         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5587                                   attr, items, actions, true, error);
5588 }
5589
5590 /**
5591  * Destroy a flow in a list.
5592  *
5593  * @param dev
5594  *   Pointer to Ethernet device.
5595  * @param list
5596  *   Pointer to the Indexed flow list. If this parameter NULL,
5597  *   there is no flow removal from the list. Be noted that as
5598  *   flow is add to the indexed list, memory of the indexed
5599  *   list points to maybe changed as flow destroyed.
5600  * @param[in] flow_idx
5601  *   Index of flow to destroy.
5602  */
5603 static void
5604 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5605                   uint32_t flow_idx)
5606 {
5607         struct mlx5_priv *priv = dev->data->dev_private;
5608         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5609                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5610
5611         if (!flow)
5612                 return;
5613         /*
5614          * Update RX queue flags only if port is started, otherwise it is
5615          * already clean.
5616          */
5617         if (dev->data->dev_started)
5618                 flow_rxq_flags_trim(dev, flow);
5619         flow_drv_destroy(dev, flow);
5620         if (list) {
5621                 rte_spinlock_lock(&priv->flow_list_lock);
5622                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5623                              flow_idx, flow, next);
5624                 rte_spinlock_unlock(&priv->flow_list_lock);
5625         }
5626         if (flow->tunnel) {
5627                 struct mlx5_flow_tunnel *tunnel;
5628
5629                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5630                 RTE_VERIFY(tunnel);
5631                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5632                         mlx5_flow_tunnel_free(dev, tunnel);
5633         }
5634         flow_mreg_del_copy_action(dev, flow);
5635         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5636 }
5637
5638 /**
5639  * Destroy all flows.
5640  *
5641  * @param dev
5642  *   Pointer to Ethernet device.
5643  * @param list
5644  *   Pointer to the Indexed flow list.
5645  * @param active
5646  *   If flushing is called avtively.
5647  */
5648 void
5649 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5650 {
5651         uint32_t num_flushed = 0;
5652
5653         while (*list) {
5654                 flow_list_destroy(dev, list, *list);
5655                 num_flushed++;
5656         }
5657         if (active) {
5658                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5659                         dev->data->port_id, num_flushed);
5660         }
5661 }
5662
5663 /**
5664  * Stop all default actions for flows.
5665  *
5666  * @param dev
5667  *   Pointer to Ethernet device.
5668  */
5669 void
5670 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5671 {
5672         flow_mreg_del_default_copy_action(dev);
5673         flow_rxq_flags_clear(dev);
5674 }
5675
5676 /**
5677  * Start all default actions for flows.
5678  *
5679  * @param dev
5680  *   Pointer to Ethernet device.
5681  * @return
5682  *   0 on success, a negative errno value otherwise and rte_errno is set.
5683  */
5684 int
5685 mlx5_flow_start_default(struct rte_eth_dev *dev)
5686 {
5687         struct rte_flow_error error;
5688
5689         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5690         return flow_mreg_add_default_copy_action(dev, &error);
5691 }
5692
5693 /**
5694  * Release key of thread specific flow workspace data.
5695  */
5696 static void
5697 flow_release_workspace(void *data)
5698 {
5699         struct mlx5_flow_workspace *wks = data;
5700         struct mlx5_flow_workspace *next;
5701
5702         while (wks) {
5703                 next = wks->next;
5704                 free(wks->rss_desc.queue);
5705                 free(wks);
5706                 wks = next;
5707         }
5708 }
5709
5710 /**
5711  * Initialize key of thread specific flow workspace data.
5712  */
5713 static void
5714 flow_alloc_workspace(void)
5715 {
5716         if (pthread_key_create(&key_workspace, flow_release_workspace))
5717                 DRV_LOG(ERR, "Can't create flow workspace data thread key.");
5718 }
5719
5720 /**
5721  * Get thread specific current flow workspace.
5722  *
5723  * @return pointer to thread specific flow workspace data, NULL on error.
5724  */
5725 struct mlx5_flow_workspace*
5726 mlx5_flow_get_thread_workspace(void)
5727 {
5728         struct mlx5_flow_workspace *data;
5729
5730         data = pthread_getspecific(key_workspace);
5731         MLX5_ASSERT(data && data->inuse);
5732         if (!data || !data->inuse)
5733                 DRV_LOG(ERR, "flow workspace not initialized.");
5734         return data;
5735 }
5736
5737 /**
5738  * Allocate and init new flow workspace.
5739  *
5740  * @return pointer to flow workspace data, NULL on error.
5741  */
5742 static struct mlx5_flow_workspace*
5743 flow_alloc_thread_workspace(void)
5744 {
5745         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
5746
5747         if (!data) {
5748                 DRV_LOG(ERR, "Failed to allocate flow workspace "
5749                         "memory.");
5750                 return NULL;
5751         }
5752         data->rss_desc.queue = calloc(1,
5753                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
5754         if (!data->rss_desc.queue)
5755                 goto err;
5756         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
5757         return data;
5758 err:
5759         if (data->rss_desc.queue)
5760                 free(data->rss_desc.queue);
5761         free(data);
5762         return NULL;
5763 }
5764
5765 /**
5766  * Get new thread specific flow workspace.
5767  *
5768  * If current workspace inuse, create new one and set as current.
5769  *
5770  * @return pointer to thread specific flow workspace data, NULL on error.
5771  */
5772 static struct mlx5_flow_workspace*
5773 mlx5_flow_push_thread_workspace(void)
5774 {
5775         struct mlx5_flow_workspace *curr;
5776         struct mlx5_flow_workspace *data;
5777
5778         if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
5779                 DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
5780                 return NULL;
5781         }
5782         curr = pthread_getspecific(key_workspace);
5783         if (!curr) {
5784                 data = flow_alloc_thread_workspace();
5785                 if (!data)
5786                         return NULL;
5787         } else if (!curr->inuse) {
5788                 data = curr;
5789         } else if (curr->next) {
5790                 data = curr->next;
5791         } else {
5792                 data = flow_alloc_thread_workspace();
5793                 if (!data)
5794                         return NULL;
5795                 curr->next = data;
5796                 data->prev = curr;
5797         }
5798         data->inuse = 1;
5799         data->flow_idx = 0;
5800         /* Set as current workspace */
5801         if (pthread_setspecific(key_workspace, data))
5802                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5803         return data;
5804 }
5805
5806 /**
5807  * Close current thread specific flow workspace.
5808  *
5809  * If previous workspace available, set it as current.
5810  *
5811  * @return pointer to thread specific flow workspace data, NULL on error.
5812  */
5813 static void
5814 mlx5_flow_pop_thread_workspace(void)
5815 {
5816         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
5817
5818         if (!data)
5819                 return;
5820         if (!data->inuse) {
5821                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
5822                 return;
5823         }
5824         data->inuse = 0;
5825         if (!data->prev)
5826                 return;
5827         if (pthread_setspecific(key_workspace, data->prev))
5828                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5829 }
5830
5831 /**
5832  * Verify the flow list is empty
5833  *
5834  * @param dev
5835  *  Pointer to Ethernet device.
5836  *
5837  * @return the number of flows not released.
5838  */
5839 int
5840 mlx5_flow_verify(struct rte_eth_dev *dev)
5841 {
5842         struct mlx5_priv *priv = dev->data->dev_private;
5843         struct rte_flow *flow;
5844         uint32_t idx;
5845         int ret = 0;
5846
5847         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5848                       flow, next) {
5849                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5850                         dev->data->port_id, (void *)flow);
5851                 ++ret;
5852         }
5853         return ret;
5854 }
5855
5856 /**
5857  * Enable default hairpin egress flow.
5858  *
5859  * @param dev
5860  *   Pointer to Ethernet device.
5861  * @param queue
5862  *   The queue index.
5863  *
5864  * @return
5865  *   0 on success, a negative errno value otherwise and rte_errno is set.
5866  */
5867 int
5868 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5869                             uint32_t queue)
5870 {
5871         struct mlx5_priv *priv = dev->data->dev_private;
5872         const struct rte_flow_attr attr = {
5873                 .egress = 1,
5874                 .priority = 0,
5875         };
5876         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5877                 .queue = queue,
5878         };
5879         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5880                 .queue = UINT32_MAX,
5881         };
5882         struct rte_flow_item items[] = {
5883                 {
5884                         .type = (enum rte_flow_item_type)
5885                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5886                         .spec = &queue_spec,
5887                         .last = NULL,
5888                         .mask = &queue_mask,
5889                 },
5890                 {
5891                         .type = RTE_FLOW_ITEM_TYPE_END,
5892                 },
5893         };
5894         struct rte_flow_action_jump jump = {
5895                 .group = MLX5_HAIRPIN_TX_TABLE,
5896         };
5897         struct rte_flow_action actions[2];
5898         uint32_t flow_idx;
5899         struct rte_flow_error error;
5900
5901         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5902         actions[0].conf = &jump;
5903         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5904         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5905                                 &attr, items, actions, false, &error);
5906         if (!flow_idx) {
5907                 DRV_LOG(DEBUG,
5908                         "Failed to create ctrl flow: rte_errno(%d),"
5909                         " type(%d), message(%s)",
5910                         rte_errno, error.type,
5911                         error.message ? error.message : " (no stated reason)");
5912                 return -rte_errno;
5913         }
5914         return 0;
5915 }
5916
5917 /**
5918  * Enable a control flow configured from the control plane.
5919  *
5920  * @param dev
5921  *   Pointer to Ethernet device.
5922  * @param eth_spec
5923  *   An Ethernet flow spec to apply.
5924  * @param eth_mask
5925  *   An Ethernet flow mask to apply.
5926  * @param vlan_spec
5927  *   A VLAN flow spec to apply.
5928  * @param vlan_mask
5929  *   A VLAN flow mask to apply.
5930  *
5931  * @return
5932  *   0 on success, a negative errno value otherwise and rte_errno is set.
5933  */
5934 int
5935 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5936                     struct rte_flow_item_eth *eth_spec,
5937                     struct rte_flow_item_eth *eth_mask,
5938                     struct rte_flow_item_vlan *vlan_spec,
5939                     struct rte_flow_item_vlan *vlan_mask)
5940 {
5941         struct mlx5_priv *priv = dev->data->dev_private;
5942         const struct rte_flow_attr attr = {
5943                 .ingress = 1,
5944                 .priority = MLX5_FLOW_PRIO_RSVD,
5945         };
5946         struct rte_flow_item items[] = {
5947                 {
5948                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5949                         .spec = eth_spec,
5950                         .last = NULL,
5951                         .mask = eth_mask,
5952                 },
5953                 {
5954                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5955                                               RTE_FLOW_ITEM_TYPE_END,
5956                         .spec = vlan_spec,
5957                         .last = NULL,
5958                         .mask = vlan_mask,
5959                 },
5960                 {
5961                         .type = RTE_FLOW_ITEM_TYPE_END,
5962                 },
5963         };
5964         uint16_t queue[priv->reta_idx_n];
5965         struct rte_flow_action_rss action_rss = {
5966                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5967                 .level = 0,
5968                 .types = priv->rss_conf.rss_hf,
5969                 .key_len = priv->rss_conf.rss_key_len,
5970                 .queue_num = priv->reta_idx_n,
5971                 .key = priv->rss_conf.rss_key,
5972                 .queue = queue,
5973         };
5974         struct rte_flow_action actions[] = {
5975                 {
5976                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5977                         .conf = &action_rss,
5978                 },
5979                 {
5980                         .type = RTE_FLOW_ACTION_TYPE_END,
5981                 },
5982         };
5983         uint32_t flow_idx;
5984         struct rte_flow_error error;
5985         unsigned int i;
5986
5987         if (!priv->reta_idx_n || !priv->rxqs_n) {
5988                 return 0;
5989         }
5990         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5991                 action_rss.types = 0;
5992         for (i = 0; i != priv->reta_idx_n; ++i)
5993                 queue[i] = (*priv->reta_idx)[i];
5994         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5995                                 &attr, items, actions, false, &error);
5996         if (!flow_idx)
5997                 return -rte_errno;
5998         return 0;
5999 }
6000
6001 /**
6002  * Enable a flow control configured from the control plane.
6003  *
6004  * @param dev
6005  *   Pointer to Ethernet device.
6006  * @param eth_spec
6007  *   An Ethernet flow spec to apply.
6008  * @param eth_mask
6009  *   An Ethernet flow mask to apply.
6010  *
6011  * @return
6012  *   0 on success, a negative errno value otherwise and rte_errno is set.
6013  */
6014 int
6015 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6016                struct rte_flow_item_eth *eth_spec,
6017                struct rte_flow_item_eth *eth_mask)
6018 {
6019         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6020 }
6021
6022 /**
6023  * Create default miss flow rule matching lacp traffic
6024  *
6025  * @param dev
6026  *   Pointer to Ethernet device.
6027  * @param eth_spec
6028  *   An Ethernet flow spec to apply.
6029  *
6030  * @return
6031  *   0 on success, a negative errno value otherwise and rte_errno is set.
6032  */
6033 int
6034 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6035 {
6036         struct mlx5_priv *priv = dev->data->dev_private;
6037         /*
6038          * The LACP matching is done by only using ether type since using
6039          * a multicast dst mac causes kernel to give low priority to this flow.
6040          */
6041         static const struct rte_flow_item_eth lacp_spec = {
6042                 .type = RTE_BE16(0x8809),
6043         };
6044         static const struct rte_flow_item_eth lacp_mask = {
6045                 .type = 0xffff,
6046         };
6047         const struct rte_flow_attr attr = {
6048                 .ingress = 1,
6049         };
6050         struct rte_flow_item items[] = {
6051                 {
6052                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6053                         .spec = &lacp_spec,
6054                         .mask = &lacp_mask,
6055                 },
6056                 {
6057                         .type = RTE_FLOW_ITEM_TYPE_END,
6058                 },
6059         };
6060         struct rte_flow_action actions[] = {
6061                 {
6062                         .type = (enum rte_flow_action_type)
6063                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6064                 },
6065                 {
6066                         .type = RTE_FLOW_ACTION_TYPE_END,
6067                 },
6068         };
6069         struct rte_flow_error error;
6070         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6071                                 &attr, items, actions, false, &error);
6072
6073         if (!flow_idx)
6074                 return -rte_errno;
6075         return 0;
6076 }
6077
6078 /**
6079  * Destroy a flow.
6080  *
6081  * @see rte_flow_destroy()
6082  * @see rte_flow_ops
6083  */
6084 int
6085 mlx5_flow_destroy(struct rte_eth_dev *dev,
6086                   struct rte_flow *flow,
6087                   struct rte_flow_error *error __rte_unused)
6088 {
6089         struct mlx5_priv *priv = dev->data->dev_private;
6090
6091         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6092         return 0;
6093 }
6094
6095 /**
6096  * Destroy all flows.
6097  *
6098  * @see rte_flow_flush()
6099  * @see rte_flow_ops
6100  */
6101 int
6102 mlx5_flow_flush(struct rte_eth_dev *dev,
6103                 struct rte_flow_error *error __rte_unused)
6104 {
6105         struct mlx5_priv *priv = dev->data->dev_private;
6106
6107         mlx5_flow_list_flush(dev, &priv->flows, false);
6108         return 0;
6109 }
6110
6111 /**
6112  * Isolated mode.
6113  *
6114  * @see rte_flow_isolate()
6115  * @see rte_flow_ops
6116  */
6117 int
6118 mlx5_flow_isolate(struct rte_eth_dev *dev,
6119                   int enable,
6120                   struct rte_flow_error *error)
6121 {
6122         struct mlx5_priv *priv = dev->data->dev_private;
6123
6124         if (dev->data->dev_started) {
6125                 rte_flow_error_set(error, EBUSY,
6126                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6127                                    NULL,
6128                                    "port must be stopped first");
6129                 return -rte_errno;
6130         }
6131         priv->isolated = !!enable;
6132         if (enable)
6133                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6134         else
6135                 dev->dev_ops = &mlx5_os_dev_ops;
6136
6137         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6138         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6139
6140         return 0;
6141 }
6142
6143 /**
6144  * Query a flow.
6145  *
6146  * @see rte_flow_query()
6147  * @see rte_flow_ops
6148  */
6149 static int
6150 flow_drv_query(struct rte_eth_dev *dev,
6151                uint32_t flow_idx,
6152                const struct rte_flow_action *actions,
6153                void *data,
6154                struct rte_flow_error *error)
6155 {
6156         struct mlx5_priv *priv = dev->data->dev_private;
6157         const struct mlx5_flow_driver_ops *fops;
6158         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6159                                                [MLX5_IPOOL_RTE_FLOW],
6160                                                flow_idx);
6161         enum mlx5_flow_drv_type ftype;
6162
6163         if (!flow) {
6164                 return rte_flow_error_set(error, ENOENT,
6165                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6166                           NULL,
6167                           "invalid flow handle");
6168         }
6169         ftype = flow->drv_type;
6170         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6171         fops = flow_get_drv_ops(ftype);
6172
6173         return fops->query(dev, flow, actions, data, error);
6174 }
6175
6176 /**
6177  * Query a flow.
6178  *
6179  * @see rte_flow_query()
6180  * @see rte_flow_ops
6181  */
6182 int
6183 mlx5_flow_query(struct rte_eth_dev *dev,
6184                 struct rte_flow *flow,
6185                 const struct rte_flow_action *actions,
6186                 void *data,
6187                 struct rte_flow_error *error)
6188 {
6189         int ret;
6190
6191         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6192                              error);
6193         if (ret < 0)
6194                 return ret;
6195         return 0;
6196 }
6197
6198 /**
6199  * Manage filter operations.
6200  *
6201  * @param dev
6202  *   Pointer to Ethernet device structure.
6203  * @param filter_type
6204  *   Filter type.
6205  * @param filter_op
6206  *   Operation to perform.
6207  * @param arg
6208  *   Pointer to operation-specific structure.
6209  *
6210  * @return
6211  *   0 on success, a negative errno value otherwise and rte_errno is set.
6212  */
6213 int
6214 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6215                      enum rte_filter_type filter_type,
6216                      enum rte_filter_op filter_op,
6217                      void *arg)
6218 {
6219         switch (filter_type) {
6220         case RTE_ETH_FILTER_GENERIC:
6221                 if (filter_op != RTE_ETH_FILTER_GET) {
6222                         rte_errno = EINVAL;
6223                         return -rte_errno;
6224                 }
6225                 *(const void **)arg = &mlx5_flow_ops;
6226                 return 0;
6227         default:
6228                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6229                         dev->data->port_id, filter_type);
6230                 rte_errno = ENOTSUP;
6231                 return -rte_errno;
6232         }
6233         return 0;
6234 }
6235
6236 /**
6237  * Create the needed meter and suffix tables.
6238  *
6239  * @param[in] dev
6240  *   Pointer to Ethernet device.
6241  * @param[in] fm
6242  *   Pointer to the flow meter.
6243  *
6244  * @return
6245  *   Pointer to table set on success, NULL otherwise.
6246  */
6247 struct mlx5_meter_domains_infos *
6248 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6249                           const struct mlx5_flow_meter *fm)
6250 {
6251         const struct mlx5_flow_driver_ops *fops;
6252
6253         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6254         return fops->create_mtr_tbls(dev, fm);
6255 }
6256
6257 /**
6258  * Destroy the meter table set.
6259  *
6260  * @param[in] dev
6261  *   Pointer to Ethernet device.
6262  * @param[in] tbl
6263  *   Pointer to the meter table set.
6264  *
6265  * @return
6266  *   0 on success.
6267  */
6268 int
6269 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6270                            struct mlx5_meter_domains_infos *tbls)
6271 {
6272         const struct mlx5_flow_driver_ops *fops;
6273
6274         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6275         return fops->destroy_mtr_tbls(dev, tbls);
6276 }
6277
6278 /**
6279  * Create policer rules.
6280  *
6281  * @param[in] dev
6282  *   Pointer to Ethernet device.
6283  * @param[in] fm
6284  *   Pointer to flow meter structure.
6285  * @param[in] attr
6286  *   Pointer to flow attributes.
6287  *
6288  * @return
6289  *   0 on success, -1 otherwise.
6290  */
6291 int
6292 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6293                                struct mlx5_flow_meter *fm,
6294                                const struct rte_flow_attr *attr)
6295 {
6296         const struct mlx5_flow_driver_ops *fops;
6297
6298         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6299         return fops->create_policer_rules(dev, fm, attr);
6300 }
6301
6302 /**
6303  * Destroy policer rules.
6304  *
6305  * @param[in] fm
6306  *   Pointer to flow meter structure.
6307  * @param[in] attr
6308  *   Pointer to flow attributes.
6309  *
6310  * @return
6311  *   0 on success, -1 otherwise.
6312  */
6313 int
6314 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6315                                 struct mlx5_flow_meter *fm,
6316                                 const struct rte_flow_attr *attr)
6317 {
6318         const struct mlx5_flow_driver_ops *fops;
6319
6320         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6321         return fops->destroy_policer_rules(dev, fm, attr);
6322 }
6323
6324 /**
6325  * Allocate a counter.
6326  *
6327  * @param[in] dev
6328  *   Pointer to Ethernet device structure.
6329  *
6330  * @return
6331  *   Index to allocated counter  on success, 0 otherwise.
6332  */
6333 uint32_t
6334 mlx5_counter_alloc(struct rte_eth_dev *dev)
6335 {
6336         const struct mlx5_flow_driver_ops *fops;
6337         struct rte_flow_attr attr = { .transfer = 0 };
6338
6339         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6340                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6341                 return fops->counter_alloc(dev);
6342         }
6343         DRV_LOG(ERR,
6344                 "port %u counter allocate is not supported.",
6345                  dev->data->port_id);
6346         return 0;
6347 }
6348
6349 /**
6350  * Free a counter.
6351  *
6352  * @param[in] dev
6353  *   Pointer to Ethernet device structure.
6354  * @param[in] cnt
6355  *   Index to counter to be free.
6356  */
6357 void
6358 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6359 {
6360         const struct mlx5_flow_driver_ops *fops;
6361         struct rte_flow_attr attr = { .transfer = 0 };
6362
6363         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6364                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6365                 fops->counter_free(dev, cnt);
6366                 return;
6367         }
6368         DRV_LOG(ERR,
6369                 "port %u counter free is not supported.",
6370                  dev->data->port_id);
6371 }
6372
6373 /**
6374  * Query counter statistics.
6375  *
6376  * @param[in] dev
6377  *   Pointer to Ethernet device structure.
6378  * @param[in] cnt
6379  *   Index to counter to query.
6380  * @param[in] clear
6381  *   Set to clear counter statistics.
6382  * @param[out] pkts
6383  *   The counter hits packets number to save.
6384  * @param[out] bytes
6385  *   The counter hits bytes number to save.
6386  *
6387  * @return
6388  *   0 on success, a negative errno value otherwise.
6389  */
6390 int
6391 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6392                    bool clear, uint64_t *pkts, uint64_t *bytes)
6393 {
6394         const struct mlx5_flow_driver_ops *fops;
6395         struct rte_flow_attr attr = { .transfer = 0 };
6396
6397         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6398                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6399                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6400         }
6401         DRV_LOG(ERR,
6402                 "port %u counter query is not supported.",
6403                  dev->data->port_id);
6404         return -ENOTSUP;
6405 }
6406
6407 /**
6408  * Allocate a new memory for the counter values wrapped by all the needed
6409  * management.
6410  *
6411  * @param[in] sh
6412  *   Pointer to mlx5_dev_ctx_shared object.
6413  *
6414  * @return
6415  *   0 on success, a negative errno value otherwise.
6416  */
6417 static int
6418 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
6419 {
6420         struct mlx5_devx_mkey_attr mkey_attr;
6421         struct mlx5_counter_stats_mem_mng *mem_mng;
6422         volatile struct flow_counter_stats *raw_data;
6423         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
6424         int size = (sizeof(struct flow_counter_stats) *
6425                         MLX5_COUNTERS_PER_POOL +
6426                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
6427                         sizeof(struct mlx5_counter_stats_mem_mng);
6428         size_t pgsize = rte_mem_page_size();
6429         uint8_t *mem;
6430         int i;
6431
6432         if (pgsize == (size_t)-1) {
6433                 DRV_LOG(ERR, "Failed to get mem page size");
6434                 rte_errno = ENOMEM;
6435                 return -ENOMEM;
6436         }
6437         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
6438         if (!mem) {
6439                 rte_errno = ENOMEM;
6440                 return -ENOMEM;
6441         }
6442         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
6443         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
6444         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
6445                                                  IBV_ACCESS_LOCAL_WRITE);
6446         if (!mem_mng->umem) {
6447                 rte_errno = errno;
6448                 mlx5_free(mem);
6449                 return -rte_errno;
6450         }
6451         mkey_attr.addr = (uintptr_t)mem;
6452         mkey_attr.size = size;
6453         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
6454         mkey_attr.pd = sh->pdn;
6455         mkey_attr.log_entity_size = 0;
6456         mkey_attr.pg_access = 0;
6457         mkey_attr.klm_array = NULL;
6458         mkey_attr.klm_num = 0;
6459         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
6460         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
6461         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
6462         if (!mem_mng->dm) {
6463                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
6464                 rte_errno = errno;
6465                 mlx5_free(mem);
6466                 return -rte_errno;
6467         }
6468         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
6469         raw_data = (volatile struct flow_counter_stats *)mem;
6470         for (i = 0; i < raws_n; ++i) {
6471                 mem_mng->raws[i].mem_mng = mem_mng;
6472                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
6473         }
6474         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
6475                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
6476                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
6477                                  next);
6478         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
6479         sh->cmng.mem_mng = mem_mng;
6480         return 0;
6481 }
6482
6483 /**
6484  * Set the statistic memory to the new counter pool.
6485  *
6486  * @param[in] sh
6487  *   Pointer to mlx5_dev_ctx_shared object.
6488  * @param[in] pool
6489  *   Pointer to the pool to set the statistic memory.
6490  *
6491  * @return
6492  *   0 on success, a negative errno value otherwise.
6493  */
6494 static int
6495 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
6496                                struct mlx5_flow_counter_pool *pool)
6497 {
6498         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6499         /* Resize statistic memory once used out. */
6500         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
6501             mlx5_flow_create_counter_stat_mem_mng(sh)) {
6502                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
6503                 return -1;
6504         }
6505         rte_spinlock_lock(&pool->sl);
6506         pool->raw = cmng->mem_mng->raws + pool->index %
6507                     MLX5_CNT_CONTAINER_RESIZE;
6508         rte_spinlock_unlock(&pool->sl);
6509         pool->raw_hw = NULL;
6510         return 0;
6511 }
6512
6513 #define MLX5_POOL_QUERY_FREQ_US 1000000
6514
6515 /**
6516  * Set the periodic procedure for triggering asynchronous batch queries for all
6517  * the counter pools.
6518  *
6519  * @param[in] sh
6520  *   Pointer to mlx5_dev_ctx_shared object.
6521  */
6522 void
6523 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6524 {
6525         uint32_t pools_n, us;
6526
6527         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
6528         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6529         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6530         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6531                 sh->cmng.query_thread_on = 0;
6532                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6533         } else {
6534                 sh->cmng.query_thread_on = 1;
6535         }
6536 }
6537
6538 /**
6539  * The periodic procedure for triggering asynchronous batch queries for all the
6540  * counter pools. This function is probably called by the host thread.
6541  *
6542  * @param[in] arg
6543  *   The parameter for the alarm process.
6544  */
6545 void
6546 mlx5_flow_query_alarm(void *arg)
6547 {
6548         struct mlx5_dev_ctx_shared *sh = arg;
6549         int ret;
6550         uint16_t pool_index = sh->cmng.pool_index;
6551         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6552         struct mlx5_flow_counter_pool *pool;
6553         uint16_t n_valid;
6554
6555         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6556                 goto set_alarm;
6557         rte_spinlock_lock(&cmng->pool_update_sl);
6558         pool = cmng->pools[pool_index];
6559         n_valid = cmng->n_valid;
6560         rte_spinlock_unlock(&cmng->pool_update_sl);
6561         /* Set the statistic memory to the new created pool. */
6562         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
6563                 goto set_alarm;
6564         if (pool->raw_hw)
6565                 /* There is a pool query in progress. */
6566                 goto set_alarm;
6567         pool->raw_hw =
6568                 LIST_FIRST(&sh->cmng.free_stat_raws);
6569         if (!pool->raw_hw)
6570                 /* No free counter statistics raw memory. */
6571                 goto set_alarm;
6572         /*
6573          * Identify the counters released between query trigger and query
6574          * handle more efficiently. The counter released in this gap period
6575          * should wait for a new round of query as the new arrived packets
6576          * will not be taken into account.
6577          */
6578         pool->query_gen++;
6579         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
6580                                                MLX5_COUNTERS_PER_POOL,
6581                                                NULL, NULL,
6582                                                pool->raw_hw->mem_mng->dm->id,
6583                                                (void *)(uintptr_t)
6584                                                pool->raw_hw->data,
6585                                                sh->devx_comp,
6586                                                (uint64_t)(uintptr_t)pool);
6587         if (ret) {
6588                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6589                         " %d", pool->min_dcs->id);
6590                 pool->raw_hw = NULL;
6591                 goto set_alarm;
6592         }
6593         LIST_REMOVE(pool->raw_hw, next);
6594         sh->cmng.pending_queries++;
6595         pool_index++;
6596         if (pool_index >= n_valid)
6597                 pool_index = 0;
6598 set_alarm:
6599         sh->cmng.pool_index = pool_index;
6600         mlx5_set_query_alarm(sh);
6601 }
6602
6603 /**
6604  * Check and callback event for new aged flow in the counter pool
6605  *
6606  * @param[in] sh
6607  *   Pointer to mlx5_dev_ctx_shared object.
6608  * @param[in] pool
6609  *   Pointer to Current counter pool.
6610  */
6611 static void
6612 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6613                    struct mlx5_flow_counter_pool *pool)
6614 {
6615         struct mlx5_priv *priv;
6616         struct mlx5_flow_counter *cnt;
6617         struct mlx5_age_info *age_info;
6618         struct mlx5_age_param *age_param;
6619         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6620         struct mlx5_counter_stats_raw *prev = pool->raw;
6621         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
6622         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
6623         uint16_t expected = AGE_CANDIDATE;
6624         uint32_t i;
6625
6626         pool->time_of_last_age_check = curr_time;
6627         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6628                 cnt = MLX5_POOL_GET_CNT(pool, i);
6629                 age_param = MLX5_CNT_TO_AGE(cnt);
6630                 if (__atomic_load_n(&age_param->state,
6631                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
6632                         continue;
6633                 if (cur->data[i].hits != prev->data[i].hits) {
6634                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
6635                                          __ATOMIC_RELAXED);
6636                         continue;
6637                 }
6638                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
6639                                        time_delta,
6640                                        __ATOMIC_RELAXED) <= age_param->timeout)
6641                         continue;
6642                 /**
6643                  * Hold the lock first, or if between the
6644                  * state AGE_TMOUT and tailq operation the
6645                  * release happened, the release procedure
6646                  * may delete a non-existent tailq node.
6647                  */
6648                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6649                 age_info = GET_PORT_AGE_INFO(priv);
6650                 rte_spinlock_lock(&age_info->aged_sl);
6651                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
6652                                                 AGE_TMOUT, false,
6653                                                 __ATOMIC_RELAXED,
6654                                                 __ATOMIC_RELAXED)) {
6655                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6656                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6657                 }
6658                 rte_spinlock_unlock(&age_info->aged_sl);
6659         }
6660         mlx5_age_event_prepare(sh);
6661 }
6662
6663 /**
6664  * Handler for the HW respond about ready values from an asynchronous batch
6665  * query. This function is probably called by the host thread.
6666  *
6667  * @param[in] sh
6668  *   The pointer to the shared device context.
6669  * @param[in] async_id
6670  *   The Devx async ID.
6671  * @param[in] status
6672  *   The status of the completion.
6673  */
6674 void
6675 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6676                                   uint64_t async_id, int status)
6677 {
6678         struct mlx5_flow_counter_pool *pool =
6679                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6680         struct mlx5_counter_stats_raw *raw_to_free;
6681         uint8_t query_gen = pool->query_gen ^ 1;
6682         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6683         enum mlx5_counter_type cnt_type =
6684                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6685                                 MLX5_COUNTER_TYPE_ORIGIN;
6686
6687         if (unlikely(status)) {
6688                 raw_to_free = pool->raw_hw;
6689         } else {
6690                 raw_to_free = pool->raw;
6691                 if (pool->is_aged)
6692                         mlx5_flow_aging_check(sh, pool);
6693                 rte_spinlock_lock(&pool->sl);
6694                 pool->raw = pool->raw_hw;
6695                 rte_spinlock_unlock(&pool->sl);
6696                 /* Be sure the new raw counters data is updated in memory. */
6697                 rte_io_wmb();
6698                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6699                         rte_spinlock_lock(&cmng->csl[cnt_type]);
6700                         TAILQ_CONCAT(&cmng->counters[cnt_type],
6701                                      &pool->counters[query_gen], next);
6702                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6703                 }
6704         }
6705         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6706         pool->raw_hw = NULL;
6707         sh->cmng.pending_queries--;
6708 }
6709
6710 static int
6711 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
6712                     const struct flow_grp_info *grp_info,
6713                     struct rte_flow_error *error)
6714 {
6715         if (grp_info->transfer && grp_info->external &&
6716             grp_info->fdb_def_rule) {
6717                 if (group == UINT32_MAX)
6718                         return rte_flow_error_set
6719                                                 (error, EINVAL,
6720                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6721                                                  NULL,
6722                                                  "group index not supported");
6723                 *table = group + 1;
6724         } else {
6725                 *table = group;
6726         }
6727         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
6728         return 0;
6729 }
6730
6731 /**
6732  * Translate the rte_flow group index to HW table value.
6733  *
6734  * If tunnel offload is disabled, all group ids converted to flow table
6735  * id using the standard method.
6736  * If tunnel offload is enabled, group id can be converted using the
6737  * standard or tunnel conversion method. Group conversion method
6738  * selection depends on flags in `grp_info` parameter:
6739  * - Internal (grp_info.external == 0) groups conversion uses the
6740  *   standard method.
6741  * - Group ids in JUMP action converted with the tunnel conversion.
6742  * - Group id in rule attribute conversion depends on a rule type and
6743  *   group id value:
6744  *   ** non zero group attributes converted with the tunnel method
6745  *   ** zero group attribute in non-tunnel rule is converted using the
6746  *      standard method - there's only one root table
6747  *   ** zero group attribute in steer tunnel rule is converted with the
6748  *      standard method - single root table
6749  *   ** zero group attribute in match tunnel rule is a special OvS
6750  *      case: that value is used for portability reasons. That group
6751  *      id is converted with the tunnel conversion method.
6752  *
6753  * @param[in] dev
6754  *   Port device
6755  * @param[in] tunnel
6756  *   PMD tunnel offload object
6757  * @param[in] group
6758  *   rte_flow group index value.
6759  * @param[out] table
6760  *   HW table value.
6761  * @param[in] grp_info
6762  *   flags used for conversion
6763  * @param[out] error
6764  *   Pointer to error structure.
6765  *
6766  * @return
6767  *   0 on success, a negative errno value otherwise and rte_errno is set.
6768  */
6769 int
6770 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
6771                          const struct mlx5_flow_tunnel *tunnel,
6772                          uint32_t group, uint32_t *table,
6773                          const struct flow_grp_info *grp_info,
6774                          struct rte_flow_error *error)
6775 {
6776         int ret;
6777         bool standard_translation;
6778
6779         if (!grp_info->skip_scale && grp_info->external &&
6780             group < MLX5_MAX_TABLES_EXTERNAL)
6781                 group *= MLX5_FLOW_TABLE_FACTOR;
6782         if (is_tunnel_offload_active(dev)) {
6783                 standard_translation = !grp_info->external ||
6784                                         grp_info->std_tbl_fix;
6785         } else {
6786                 standard_translation = true;
6787         }
6788         DRV_LOG(DEBUG,
6789                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
6790                 dev->data->port_id, group, grp_info->transfer,
6791                 grp_info->external, grp_info->fdb_def_rule,
6792                 standard_translation ? "STANDARD" : "TUNNEL");
6793         if (standard_translation)
6794                 ret = flow_group_to_table(dev->data->port_id, group, table,
6795                                           grp_info, error);
6796         else
6797                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
6798                                                       table, error);
6799
6800         return ret;
6801 }
6802
6803 /**
6804  * Discover availability of metadata reg_c's.
6805  *
6806  * Iteratively use test flows to check availability.
6807  *
6808  * @param[in] dev
6809  *   Pointer to the Ethernet device structure.
6810  *
6811  * @return
6812  *   0 on success, a negative errno value otherwise and rte_errno is set.
6813  */
6814 int
6815 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6816 {
6817         struct mlx5_priv *priv = dev->data->dev_private;
6818         struct mlx5_dev_config *config = &priv->config;
6819         enum modify_reg idx;
6820         int n = 0;
6821
6822         /* reg_c[0] and reg_c[1] are reserved. */
6823         config->flow_mreg_c[n++] = REG_C_0;
6824         config->flow_mreg_c[n++] = REG_C_1;
6825         /* Discover availability of other reg_c's. */
6826         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6827                 struct rte_flow_attr attr = {
6828                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6829                         .priority = MLX5_FLOW_PRIO_RSVD,
6830                         .ingress = 1,
6831                 };
6832                 struct rte_flow_item items[] = {
6833                         [0] = {
6834                                 .type = RTE_FLOW_ITEM_TYPE_END,
6835                         },
6836                 };
6837                 struct rte_flow_action actions[] = {
6838                         [0] = {
6839                                 .type = (enum rte_flow_action_type)
6840                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6841                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6842                                         .src = REG_C_1,
6843                                         .dst = idx,
6844                                 },
6845                         },
6846                         [1] = {
6847                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6848                                 .conf = &(struct rte_flow_action_jump){
6849                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6850                                 },
6851                         },
6852                         [2] = {
6853                                 .type = RTE_FLOW_ACTION_TYPE_END,
6854                         },
6855                 };
6856                 uint32_t flow_idx;
6857                 struct rte_flow *flow;
6858                 struct rte_flow_error error;
6859
6860                 if (!config->dv_flow_en)
6861                         break;
6862                 /* Create internal flow, validation skips copy action. */
6863                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6864                                             actions, false, &error);
6865                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6866                                       flow_idx);
6867                 if (!flow)
6868                         continue;
6869                 config->flow_mreg_c[n++] = idx;
6870                 flow_list_destroy(dev, NULL, flow_idx);
6871         }
6872         for (; n < MLX5_MREG_C_NUM; ++n)
6873                 config->flow_mreg_c[n] = REG_NON;
6874         return 0;
6875 }
6876
6877 /**
6878  * Dump flow raw hw data to file
6879  *
6880  * @param[in] dev
6881  *    The pointer to Ethernet device.
6882  * @param[in] file
6883  *   A pointer to a file for output.
6884  * @param[out] error
6885  *   Perform verbose error reporting if not NULL. PMDs initialize this
6886  *   structure in case of error only.
6887  * @return
6888  *   0 on success, a nagative value otherwise.
6889  */
6890 int
6891 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6892                    FILE *file,
6893                    struct rte_flow_error *error __rte_unused)
6894 {
6895         struct mlx5_priv *priv = dev->data->dev_private;
6896         struct mlx5_dev_ctx_shared *sh = priv->sh;
6897
6898         if (!priv->config.dv_flow_en) {
6899                 if (fputs("device dv flow disabled\n", file) <= 0)
6900                         return -errno;
6901                 return -ENOTSUP;
6902         }
6903         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6904                                        sh->tx_domain, file);
6905 }
6906
6907 /**
6908  * Get aged-out flows.
6909  *
6910  * @param[in] dev
6911  *   Pointer to the Ethernet device structure.
6912  * @param[in] context
6913  *   The address of an array of pointers to the aged-out flows contexts.
6914  * @param[in] nb_countexts
6915  *   The length of context array pointers.
6916  * @param[out] error
6917  *   Perform verbose error reporting if not NULL. Initialized in case of
6918  *   error only.
6919  *
6920  * @return
6921  *   how many contexts get in success, otherwise negative errno value.
6922  *   if nb_contexts is 0, return the amount of all aged contexts.
6923  *   if nb_contexts is not 0 , return the amount of aged flows reported
6924  *   in the context array.
6925  */
6926 int
6927 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6928                         uint32_t nb_contexts, struct rte_flow_error *error)
6929 {
6930         const struct mlx5_flow_driver_ops *fops;
6931         struct rte_flow_attr attr = { .transfer = 0 };
6932
6933         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6934                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6935                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6936                                                     error);
6937         }
6938         DRV_LOG(ERR,
6939                 "port %u get aged flows is not supported.",
6940                  dev->data->port_id);
6941         return -ENOTSUP;
6942 }
6943
6944 /* Wrapper for driver action_validate op callback */
6945 static int
6946 flow_drv_action_validate(struct rte_eth_dev *dev,
6947                          const struct rte_flow_shared_action_conf *conf,
6948                          const struct rte_flow_action *action,
6949                          const struct mlx5_flow_driver_ops *fops,
6950                          struct rte_flow_error *error)
6951 {
6952         static const char err_msg[] = "shared action validation unsupported";
6953
6954         if (!fops->action_validate) {
6955                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6956                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6957                                    NULL, err_msg);
6958                 return -rte_errno;
6959         }
6960         return fops->action_validate(dev, conf, action, error);
6961 }
6962
6963 /**
6964  * Destroys the shared action by handle.
6965  *
6966  * @param dev
6967  *   Pointer to Ethernet device structure.
6968  * @param[in] action
6969  *   Handle for the shared action to be destroyed.
6970  * @param[out] error
6971  *   Perform verbose error reporting if not NULL. PMDs initialize this
6972  *   structure in case of error only.
6973  *
6974  * @return
6975  *   0 on success, a negative errno value otherwise and rte_errno is set.
6976  *
6977  * @note: wrapper for driver action_create op callback.
6978  */
6979 static int
6980 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
6981                            struct rte_flow_shared_action *action,
6982                            struct rte_flow_error *error)
6983 {
6984         static const char err_msg[] = "shared action destruction unsupported";
6985         struct rte_flow_attr attr = { .transfer = 0 };
6986         const struct mlx5_flow_driver_ops *fops =
6987                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
6988
6989         if (!fops->action_destroy) {
6990                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6991                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6992                                    NULL, err_msg);
6993                 return -rte_errno;
6994         }
6995         return fops->action_destroy(dev, action, error);
6996 }
6997
6998 /* Wrapper for driver action_destroy op callback */
6999 static int
7000 flow_drv_action_update(struct rte_eth_dev *dev,
7001                        struct rte_flow_shared_action *action,
7002                        const void *action_conf,
7003                        const struct mlx5_flow_driver_ops *fops,
7004                        struct rte_flow_error *error)
7005 {
7006         static const char err_msg[] = "shared action update unsupported";
7007
7008         if (!fops->action_update) {
7009                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7010                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7011                                    NULL, err_msg);
7012                 return -rte_errno;
7013         }
7014         return fops->action_update(dev, action, action_conf, error);
7015 }
7016
7017 /* Wrapper for driver action_destroy op callback */
7018 static int
7019 flow_drv_action_query(struct rte_eth_dev *dev,
7020                       const struct rte_flow_shared_action *action,
7021                       void *data,
7022                       const struct mlx5_flow_driver_ops *fops,
7023                       struct rte_flow_error *error)
7024 {
7025         static const char err_msg[] = "shared action query unsupported";
7026
7027         if (!fops->action_query) {
7028                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7029                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7030                                    NULL, err_msg);
7031                 return -rte_errno;
7032         }
7033         return fops->action_query(dev, action, data, error);
7034 }
7035
7036 /**
7037  * Create shared action for reuse in multiple flow rules.
7038  *
7039  * @param dev
7040  *   Pointer to Ethernet device structure.
7041  * @param[in] action
7042  *   Action configuration for shared action creation.
7043  * @param[out] error
7044  *   Perform verbose error reporting if not NULL. PMDs initialize this
7045  *   structure in case of error only.
7046  * @return
7047  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7048  */
7049 static struct rte_flow_shared_action *
7050 mlx5_shared_action_create(struct rte_eth_dev *dev,
7051                           const struct rte_flow_shared_action_conf *conf,
7052                           const struct rte_flow_action *action,
7053                           struct rte_flow_error *error)
7054 {
7055         static const char err_msg[] = "shared action creation unsupported";
7056         struct rte_flow_attr attr = { .transfer = 0 };
7057         const struct mlx5_flow_driver_ops *fops =
7058                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7059
7060         if (flow_drv_action_validate(dev, conf, action, fops, error))
7061                 return NULL;
7062         if (!fops->action_create) {
7063                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7064                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7065                                    NULL, err_msg);
7066                 return NULL;
7067         }
7068         return fops->action_create(dev, conf, action, error);
7069 }
7070
7071 /**
7072  * Updates inplace the shared action configuration pointed by *action* handle
7073  * with the configuration provided as *action* argument.
7074  * The update of the shared action configuration effects all flow rules reusing
7075  * the action via handle.
7076  *
7077  * @param dev
7078  *   Pointer to Ethernet device structure.
7079  * @param[in] shared_action
7080  *   Handle for the shared action to be updated.
7081  * @param[in] action
7082  *   Action specification used to modify the action pointed by handle.
7083  *   *action* should be of same type with the action pointed by the *action*
7084  *   handle argument, otherwise considered as invalid.
7085  * @param[out] error
7086  *   Perform verbose error reporting if not NULL. PMDs initialize this
7087  *   structure in case of error only.
7088  *
7089  * @return
7090  *   0 on success, a negative errno value otherwise and rte_errno is set.
7091  */
7092 static int
7093 mlx5_shared_action_update(struct rte_eth_dev *dev,
7094                 struct rte_flow_shared_action *shared_action,
7095                 const struct rte_flow_action *action,
7096                 struct rte_flow_error *error)
7097 {
7098         struct rte_flow_attr attr = { .transfer = 0 };
7099         const struct mlx5_flow_driver_ops *fops =
7100                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7101         int ret;
7102
7103         ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7104         if (ret)
7105                 return ret;
7106         return flow_drv_action_update(dev, shared_action, action->conf, fops,
7107                                       error);
7108 }
7109
7110 /**
7111  * Query the shared action by handle.
7112  *
7113  * This function allows retrieving action-specific data such as counters.
7114  * Data is gathered by special action which may be present/referenced in
7115  * more than one flow rule definition.
7116  *
7117  * \see RTE_FLOW_ACTION_TYPE_COUNT
7118  *
7119  * @param dev
7120  *   Pointer to Ethernet device structure.
7121  * @param[in] action
7122  *   Handle for the shared action to query.
7123  * @param[in, out] data
7124  *   Pointer to storage for the associated query data type.
7125  * @param[out] error
7126  *   Perform verbose error reporting if not NULL. PMDs initialize this
7127  *   structure in case of error only.
7128  *
7129  * @return
7130  *   0 on success, a negative errno value otherwise and rte_errno is set.
7131  */
7132 static int
7133 mlx5_shared_action_query(struct rte_eth_dev *dev,
7134                          const struct rte_flow_shared_action *action,
7135                          void *data,
7136                          struct rte_flow_error *error)
7137 {
7138         struct rte_flow_attr attr = { .transfer = 0 };
7139         const struct mlx5_flow_driver_ops *fops =
7140                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7141
7142         return flow_drv_action_query(dev, action, data, fops, error);
7143 }
7144
7145 /**
7146  * Destroy all shared actions.
7147  *
7148  * @param dev
7149  *   Pointer to Ethernet device.
7150  *
7151  * @return
7152  *   0 on success, a negative errno value otherwise and rte_errno is set.
7153  */
7154 int
7155 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7156 {
7157         struct rte_flow_error error;
7158         struct mlx5_priv *priv = dev->data->dev_private;
7159         struct mlx5_shared_action_rss *action;
7160         int ret = 0;
7161         uint32_t idx;
7162
7163         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7164                       priv->rss_shared_actions, idx, action, next) {
7165                 ret |= mlx5_shared_action_destroy(dev,
7166                        (struct rte_flow_shared_action *)(uintptr_t)idx, &error);
7167         }
7168         return ret;
7169 }
7170
7171 #ifndef HAVE_MLX5DV_DR
7172 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
7173 #else
7174 #define MLX5_DOMAIN_SYNC_FLOW \
7175         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
7176 #endif
7177
7178 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
7179 {
7180         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
7181         const struct mlx5_flow_driver_ops *fops;
7182         int ret;
7183         struct rte_flow_attr attr = { .transfer = 0 };
7184
7185         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7186         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
7187         if (ret > 0)
7188                 ret = -ret;
7189         return ret;
7190 }
7191
7192 /**
7193  * tunnel offload functionalilty is defined for DV environment only
7194  */
7195 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7196 __extension__
7197 union tunnel_offload_mark {
7198         uint32_t val;
7199         struct {
7200                 uint32_t app_reserve:8;
7201                 uint32_t table_id:15;
7202                 uint32_t transfer:1;
7203                 uint32_t _unused_:8;
7204         };
7205 };
7206
7207 static bool
7208 mlx5_access_tunnel_offload_db
7209         (struct rte_eth_dev *dev,
7210          bool (*match)(struct rte_eth_dev *,
7211                        struct mlx5_flow_tunnel *, const void *),
7212          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7213          void (*miss)(struct rte_eth_dev *, void *),
7214          void *ctx, bool lock_op);
7215
7216 static int
7217 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
7218                              struct rte_flow *flow,
7219                              const struct rte_flow_attr *attr,
7220                              const struct rte_flow_action *app_actions,
7221                              uint32_t flow_idx,
7222                              struct tunnel_default_miss_ctx *ctx,
7223                              struct rte_flow_error *error)
7224 {
7225         struct mlx5_priv *priv = dev->data->dev_private;
7226         struct mlx5_flow *dev_flow;
7227         struct rte_flow_attr miss_attr = *attr;
7228         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
7229         const struct rte_flow_item miss_items[2] = {
7230                 {
7231                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7232                         .spec = NULL,
7233                         .last = NULL,
7234                         .mask = NULL
7235                 },
7236                 {
7237                         .type = RTE_FLOW_ITEM_TYPE_END,
7238                         .spec = NULL,
7239                         .last = NULL,
7240                         .mask = NULL
7241                 }
7242         };
7243         union tunnel_offload_mark mark_id;
7244         struct rte_flow_action_mark miss_mark;
7245         struct rte_flow_action miss_actions[3] = {
7246                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
7247                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
7248         };
7249         const struct rte_flow_action_jump *jump_data;
7250         uint32_t i, flow_table = 0; /* prevent compilation warning */
7251         struct flow_grp_info grp_info = {
7252                 .external = 1,
7253                 .transfer = attr->transfer,
7254                 .fdb_def_rule = !!priv->fdb_def_rule,
7255                 .std_tbl_fix = 0,
7256         };
7257         int ret;
7258
7259         if (!attr->transfer) {
7260                 uint32_t q_size;
7261
7262                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
7263                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
7264                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
7265                                          0, SOCKET_ID_ANY);
7266                 if (!ctx->queue)
7267                         return rte_flow_error_set
7268                                 (error, ENOMEM,
7269                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7270                                 NULL, "invalid default miss RSS");
7271                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7272                 ctx->action_rss.level = 0,
7273                 ctx->action_rss.types = priv->rss_conf.rss_hf,
7274                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
7275                 ctx->action_rss.queue_num = priv->reta_idx_n,
7276                 ctx->action_rss.key = priv->rss_conf.rss_key,
7277                 ctx->action_rss.queue = ctx->queue;
7278                 if (!priv->reta_idx_n || !priv->rxqs_n)
7279                         return rte_flow_error_set
7280                                 (error, EINVAL,
7281                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7282                                 NULL, "invalid port configuration");
7283                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7284                         ctx->action_rss.types = 0;
7285                 for (i = 0; i != priv->reta_idx_n; ++i)
7286                         ctx->queue[i] = (*priv->reta_idx)[i];
7287         } else {
7288                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
7289                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
7290         }
7291         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
7292         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
7293         jump_data = app_actions->conf;
7294         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
7295         miss_attr.group = jump_data->group;
7296         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
7297                                        &flow_table, &grp_info, error);
7298         if (ret)
7299                 return rte_flow_error_set(error, EINVAL,
7300                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7301                                           NULL, "invalid tunnel id");
7302         mark_id.app_reserve = 0;
7303         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
7304         mark_id.transfer = !!attr->transfer;
7305         mark_id._unused_ = 0;
7306         miss_mark.id = mark_id.val;
7307         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
7308                                     miss_items, miss_actions, flow_idx, error);
7309         if (!dev_flow)
7310                 return -rte_errno;
7311         dev_flow->flow = flow;
7312         dev_flow->external = true;
7313         dev_flow->tunnel = tunnel;
7314         /* Subflow object was created, we must include one in the list. */
7315         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7316                       dev_flow->handle, next);
7317         DRV_LOG(DEBUG,
7318                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
7319                 dev->data->port_id, tunnel->app_tunnel.type,
7320                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
7321         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
7322                                   miss_actions, error);
7323         if (!ret)
7324                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
7325                                                   error);
7326
7327         return ret;
7328 }
7329
7330 static const struct mlx5_flow_tbl_data_entry  *
7331 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7332 {
7333         struct mlx5_priv *priv = dev->data->dev_private;
7334         struct mlx5_dev_ctx_shared *sh = priv->sh;
7335         struct mlx5_hlist_entry *he;
7336         union tunnel_offload_mark mbits = { .val = mark };
7337         union mlx5_flow_tbl_key table_key = {
7338                 {
7339                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7340                         .dummy = 0,
7341                         .domain = !!mbits.transfer,
7342                         .direction = 0,
7343                 }
7344         };
7345         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
7346         return he ?
7347                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7348 }
7349
7350 static void
7351 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
7352                                    struct mlx5_hlist_entry *entry)
7353 {
7354         struct mlx5_dev_ctx_shared *sh = list->ctx;
7355         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7356
7357         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7358                         tunnel_flow_tbl_to_id(tte->flow_table));
7359         mlx5_free(tte);
7360 }
7361
7362 static struct mlx5_hlist_entry *
7363 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
7364                                    uint64_t key __rte_unused,
7365                                    void *ctx __rte_unused)
7366 {
7367         struct mlx5_dev_ctx_shared *sh = list->ctx;
7368         struct tunnel_tbl_entry *tte;
7369
7370         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7371                           sizeof(*tte), 0,
7372                           SOCKET_ID_ANY);
7373         if (!tte)
7374                 goto err;
7375         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7376                           &tte->flow_table);
7377         if (tte->flow_table >= MLX5_MAX_TABLES) {
7378                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
7379                         tte->flow_table);
7380                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7381                                 tte->flow_table);
7382                 goto err;
7383         } else if (!tte->flow_table) {
7384                 goto err;
7385         }
7386         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7387         return &tte->hash;
7388 err:
7389         if (tte)
7390                 mlx5_free(tte);
7391         return NULL;
7392 }
7393
7394 static uint32_t
7395 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7396                                 const struct mlx5_flow_tunnel *tunnel,
7397                                 uint32_t group, uint32_t *table,
7398                                 struct rte_flow_error *error)
7399 {
7400         struct mlx5_hlist_entry *he;
7401         struct tunnel_tbl_entry *tte;
7402         union tunnel_tbl_key key = {
7403                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7404                 .group = group
7405         };
7406         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7407         struct mlx5_hlist *group_hash;
7408
7409         group_hash = tunnel ? tunnel->groups : thub->groups;
7410         he = mlx5_hlist_register(group_hash, key.val, NULL);
7411         if (!he)
7412                 return rte_flow_error_set(error, EINVAL,
7413                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7414                                           NULL,
7415                                           "tunnel group index not supported");
7416         tte = container_of(he, typeof(*tte), hash);
7417         *table = tte->flow_table;
7418         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7419                 dev->data->port_id, key.tunnel_id, group, *table);
7420         return 0;
7421 }
7422
7423 static void
7424 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
7425                       struct mlx5_flow_tunnel *tunnel)
7426 {
7427         struct mlx5_priv *priv = dev->data->dev_private;
7428         struct mlx5_indexed_pool *ipool;
7429
7430         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
7431                 dev->data->port_id, tunnel->tunnel_id);
7432         mlx5_hlist_destroy(tunnel->groups);
7433         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7434         mlx5_ipool_free(ipool, tunnel->tunnel_id);
7435 }
7436
7437 static bool
7438 mlx5_access_tunnel_offload_db
7439         (struct rte_eth_dev *dev,
7440          bool (*match)(struct rte_eth_dev *,
7441                        struct mlx5_flow_tunnel *, const void *),
7442          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7443          void (*miss)(struct rte_eth_dev *, void *),
7444          void *ctx, bool lock_op)
7445 {
7446         bool verdict = false;
7447         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7448         struct mlx5_flow_tunnel *tunnel;
7449
7450         rte_spinlock_lock(&thub->sl);
7451         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
7452                 verdict = match(dev, tunnel, (const void *)ctx);
7453                 if (verdict)
7454                         break;
7455         }
7456         if (!lock_op)
7457                 rte_spinlock_unlock(&thub->sl);
7458         if (verdict && hit)
7459                 hit(dev, tunnel, ctx);
7460         if (!verdict && miss)
7461                 miss(dev, ctx);
7462         if (lock_op)
7463                 rte_spinlock_unlock(&thub->sl);
7464
7465         return verdict;
7466 }
7467
7468 struct tunnel_db_find_tunnel_id_ctx {
7469         uint32_t tunnel_id;
7470         struct mlx5_flow_tunnel *tunnel;
7471 };
7472
7473 static bool
7474 find_tunnel_id_match(struct rte_eth_dev *dev,
7475                      struct mlx5_flow_tunnel *tunnel, const void *x)
7476 {
7477         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7478
7479         RTE_SET_USED(dev);
7480         return tunnel->tunnel_id == ctx->tunnel_id;
7481 }
7482
7483 static void
7484 find_tunnel_id_hit(struct rte_eth_dev *dev,
7485                    struct mlx5_flow_tunnel *tunnel, void *x)
7486 {
7487         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7488         RTE_SET_USED(dev);
7489         ctx->tunnel = tunnel;
7490 }
7491
7492 static struct mlx5_flow_tunnel *
7493 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
7494 {
7495         struct tunnel_db_find_tunnel_id_ctx ctx = {
7496                 .tunnel_id = id,
7497         };
7498
7499         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
7500                                       find_tunnel_id_hit, NULL, &ctx, true);
7501
7502         return ctx.tunnel;
7503 }
7504
7505 static struct mlx5_flow_tunnel *
7506 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
7507                           const struct rte_flow_tunnel *app_tunnel)
7508 {
7509         struct mlx5_priv *priv = dev->data->dev_private;
7510         struct mlx5_indexed_pool *ipool;
7511         struct mlx5_flow_tunnel *tunnel;
7512         uint32_t id;
7513
7514         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7515         tunnel = mlx5_ipool_zmalloc(ipool, &id);
7516         if (!tunnel)
7517                 return NULL;
7518         if (id >= MLX5_MAX_TUNNELS) {
7519                 mlx5_ipool_free(ipool, id);
7520                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
7521                 return NULL;
7522         }
7523         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
7524                                            mlx5_flow_tunnel_grp2tbl_create_cb,
7525                                            NULL,
7526                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
7527         if (!tunnel->groups) {
7528                 mlx5_ipool_free(ipool, id);
7529                 return NULL;
7530         }
7531         tunnel->groups->ctx = priv->sh;
7532         /* initiate new PMD tunnel */
7533         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
7534         tunnel->tunnel_id = id;
7535         tunnel->action.type = (typeof(tunnel->action.type))
7536                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
7537         tunnel->action.conf = tunnel;
7538         tunnel->item.type = (typeof(tunnel->item.type))
7539                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
7540         tunnel->item.spec = tunnel;
7541         tunnel->item.last = NULL;
7542         tunnel->item.mask = NULL;
7543
7544         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
7545                 dev->data->port_id, tunnel->tunnel_id);
7546
7547         return tunnel;
7548 }
7549
7550 struct tunnel_db_get_tunnel_ctx {
7551         const struct rte_flow_tunnel *app_tunnel;
7552         struct mlx5_flow_tunnel *tunnel;
7553 };
7554
7555 static bool get_tunnel_match(struct rte_eth_dev *dev,
7556                              struct mlx5_flow_tunnel *tunnel, const void *x)
7557 {
7558         const struct tunnel_db_get_tunnel_ctx *ctx = x;
7559
7560         RTE_SET_USED(dev);
7561         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
7562                        sizeof(*ctx->app_tunnel));
7563 }
7564
7565 static void get_tunnel_hit(struct rte_eth_dev *dev,
7566                            struct mlx5_flow_tunnel *tunnel, void *x)
7567 {
7568         /* called under tunnel spinlock protection */
7569         struct tunnel_db_get_tunnel_ctx *ctx = x;
7570
7571         RTE_SET_USED(dev);
7572         tunnel->refctn++;
7573         ctx->tunnel = tunnel;
7574 }
7575
7576 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
7577 {
7578         /* called under tunnel spinlock protection */
7579         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7580         struct tunnel_db_get_tunnel_ctx *ctx = x;
7581
7582         rte_spinlock_unlock(&thub->sl);
7583         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
7584         ctx->tunnel->refctn = 1;
7585         rte_spinlock_lock(&thub->sl);
7586         if (ctx->tunnel)
7587                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
7588 }
7589
7590
7591 static int
7592 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
7593                      const struct rte_flow_tunnel *app_tunnel,
7594                      struct mlx5_flow_tunnel **tunnel)
7595 {
7596         struct tunnel_db_get_tunnel_ctx ctx = {
7597                 .app_tunnel = app_tunnel,
7598         };
7599
7600         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
7601                                       get_tunnel_miss, &ctx, true);
7602         *tunnel = ctx.tunnel;
7603         return ctx.tunnel ? 0 : -ENOMEM;
7604 }
7605
7606 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
7607 {
7608         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
7609
7610         if (!thub)
7611                 return;
7612         if (!LIST_EMPTY(&thub->tunnels))
7613                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
7614         mlx5_hlist_destroy(thub->groups);
7615         mlx5_free(thub);
7616 }
7617
7618 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
7619 {
7620         int err;
7621         struct mlx5_flow_tunnel_hub *thub;
7622
7623         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
7624                            0, SOCKET_ID_ANY);
7625         if (!thub)
7626                 return -ENOMEM;
7627         LIST_INIT(&thub->tunnels);
7628         rte_spinlock_init(&thub->sl);
7629         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
7630                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
7631                                          NULL,
7632                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
7633         if (!thub->groups) {
7634                 err = -rte_errno;
7635                 goto err;
7636         }
7637         thub->groups->ctx = sh;
7638         sh->tunnel_hub = thub;
7639
7640         return 0;
7641
7642 err:
7643         if (thub->groups)
7644                 mlx5_hlist_destroy(thub->groups);
7645         if (thub)
7646                 mlx5_free(thub);
7647         return err;
7648 }
7649
7650 static inline bool
7651 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
7652                           struct rte_flow_tunnel *tunnel,
7653                           const char *err_msg)
7654 {
7655         err_msg = NULL;
7656         if (!is_tunnel_offload_active(dev)) {
7657                 err_msg = "tunnel offload was not activated";
7658                 goto out;
7659         } else if (!tunnel) {
7660                 err_msg = "no application tunnel";
7661                 goto out;
7662         }
7663
7664         switch (tunnel->type) {
7665         default:
7666                 err_msg = "unsupported tunnel type";
7667                 goto out;
7668         case RTE_FLOW_ITEM_TYPE_VXLAN:
7669                 break;
7670         }
7671
7672 out:
7673         return !err_msg;
7674 }
7675
7676 static int
7677 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
7678                     struct rte_flow_tunnel *app_tunnel,
7679                     struct rte_flow_action **actions,
7680                     uint32_t *num_of_actions,
7681                     struct rte_flow_error *error)
7682 {
7683         int ret;
7684         struct mlx5_flow_tunnel *tunnel;
7685         const char *err_msg = NULL;
7686         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7687
7688         if (!verdict)
7689                 return rte_flow_error_set(error, EINVAL,
7690                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7691                                           err_msg);
7692         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7693         if (ret < 0) {
7694                 return rte_flow_error_set(error, ret,
7695                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7696                                           "failed to initialize pmd tunnel");
7697         }
7698         *actions = &tunnel->action;
7699         *num_of_actions = 1;
7700         return 0;
7701 }
7702
7703 static int
7704 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
7705                        struct rte_flow_tunnel *app_tunnel,
7706                        struct rte_flow_item **items,
7707                        uint32_t *num_of_items,
7708                        struct rte_flow_error *error)
7709 {
7710         int ret;
7711         struct mlx5_flow_tunnel *tunnel;
7712         const char *err_msg = NULL;
7713         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7714
7715         if (!verdict)
7716                 return rte_flow_error_set(error, EINVAL,
7717                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7718                                           err_msg);
7719         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7720         if (ret < 0) {
7721                 return rte_flow_error_set(error, ret,
7722                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7723                                           "failed to initialize pmd tunnel");
7724         }
7725         *items = &tunnel->item;
7726         *num_of_items = 1;
7727         return 0;
7728 }
7729
7730 struct tunnel_db_element_release_ctx {
7731         struct rte_flow_item *items;
7732         struct rte_flow_action *actions;
7733         uint32_t num_elements;
7734         struct rte_flow_error *error;
7735         int ret;
7736 };
7737
7738 static bool
7739 tunnel_element_release_match(struct rte_eth_dev *dev,
7740                              struct mlx5_flow_tunnel *tunnel, const void *x)
7741 {
7742         const struct tunnel_db_element_release_ctx *ctx = x;
7743
7744         RTE_SET_USED(dev);
7745         if (ctx->num_elements != 1)
7746                 return false;
7747         else if (ctx->items)
7748                 return ctx->items == &tunnel->item;
7749         else if (ctx->actions)
7750                 return ctx->actions == &tunnel->action;
7751
7752         return false;
7753 }
7754
7755 static void
7756 tunnel_element_release_hit(struct rte_eth_dev *dev,
7757                            struct mlx5_flow_tunnel *tunnel, void *x)
7758 {
7759         struct tunnel_db_element_release_ctx *ctx = x;
7760         ctx->ret = 0;
7761         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7762                 mlx5_flow_tunnel_free(dev, tunnel);
7763 }
7764
7765 static void
7766 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
7767 {
7768         struct tunnel_db_element_release_ctx *ctx = x;
7769         RTE_SET_USED(dev);
7770         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
7771                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7772                                       "invalid argument");
7773 }
7774
7775 static int
7776 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
7777                        struct rte_flow_item *pmd_items,
7778                        uint32_t num_items, struct rte_flow_error *err)
7779 {
7780         struct tunnel_db_element_release_ctx ctx = {
7781                 .items = pmd_items,
7782                 .actions = NULL,
7783                 .num_elements = num_items,
7784                 .error = err,
7785         };
7786
7787         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7788                                       tunnel_element_release_hit,
7789                                       tunnel_element_release_miss, &ctx, false);
7790
7791         return ctx.ret;
7792 }
7793
7794 static int
7795 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
7796                          struct rte_flow_action *pmd_actions,
7797                          uint32_t num_actions, struct rte_flow_error *err)
7798 {
7799         struct tunnel_db_element_release_ctx ctx = {
7800                 .items = NULL,
7801                 .actions = pmd_actions,
7802                 .num_elements = num_actions,
7803                 .error = err,
7804         };
7805
7806         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7807                                       tunnel_element_release_hit,
7808                                       tunnel_element_release_miss, &ctx, false);
7809
7810         return ctx.ret;
7811 }
7812
7813 static int
7814 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
7815                                   struct rte_mbuf *m,
7816                                   struct rte_flow_restore_info *info,
7817                                   struct rte_flow_error *err)
7818 {
7819         uint64_t ol_flags = m->ol_flags;
7820         const struct mlx5_flow_tbl_data_entry *tble;
7821         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
7822
7823         if (!is_tunnel_offload_active(dev)) {
7824                 info->flags = 0;
7825                 return 0;
7826         }
7827
7828         if ((ol_flags & mask) != mask)
7829                 goto err;
7830         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
7831         if (!tble) {
7832                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
7833                         dev->data->port_id, m->hash.fdir.hi);
7834                 goto err;
7835         }
7836         MLX5_ASSERT(tble->tunnel);
7837         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
7838         info->group_id = tble->group_id;
7839         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
7840                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
7841                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
7842
7843         return 0;
7844
7845 err:
7846         return rte_flow_error_set(err, EINVAL,
7847                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7848                                   "failed to get restore info");
7849 }
7850
7851 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
7852 static int
7853 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
7854                            __rte_unused struct rte_flow_tunnel *app_tunnel,
7855                            __rte_unused struct rte_flow_action **actions,
7856                            __rte_unused uint32_t *num_of_actions,
7857                            __rte_unused struct rte_flow_error *error)
7858 {
7859         return -ENOTSUP;
7860 }
7861
7862 static int
7863 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
7864                        __rte_unused struct rte_flow_tunnel *app_tunnel,
7865                        __rte_unused struct rte_flow_item **items,
7866                        __rte_unused uint32_t *num_of_items,
7867                        __rte_unused struct rte_flow_error *error)
7868 {
7869         return -ENOTSUP;
7870 }
7871
7872 static int
7873 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
7874                               __rte_unused struct rte_flow_item *pmd_items,
7875                               __rte_unused uint32_t num_items,
7876                               __rte_unused struct rte_flow_error *err)
7877 {
7878         return -ENOTSUP;
7879 }
7880
7881 static int
7882 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
7883                                 __rte_unused struct rte_flow_action *pmd_action,
7884                                 __rte_unused uint32_t num_actions,
7885                                 __rte_unused struct rte_flow_error *err)
7886 {
7887         return -ENOTSUP;
7888 }
7889
7890 static int
7891 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
7892                                   __rte_unused struct rte_mbuf *m,
7893                                   __rte_unused struct rte_flow_restore_info *i,
7894                                   __rte_unused struct rte_flow_error *err)
7895 {
7896         return -ENOTSUP;
7897 }
7898
7899 static int
7900 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
7901                              __rte_unused struct rte_flow *flow,
7902                              __rte_unused const struct rte_flow_attr *attr,
7903                              __rte_unused const struct rte_flow_action *actions,
7904                              __rte_unused uint32_t flow_idx,
7905                              __rte_unused struct tunnel_default_miss_ctx *ctx,
7906                              __rte_unused struct rte_flow_error *error)
7907 {
7908         return -ENOTSUP;
7909 }
7910
7911 static struct mlx5_flow_tunnel *
7912 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
7913                     __rte_unused uint32_t id)
7914 {
7915         return NULL;
7916 }
7917
7918 static void
7919 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
7920                       __rte_unused struct mlx5_flow_tunnel *tunnel)
7921 {
7922 }
7923
7924 static uint32_t
7925 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
7926                                 __rte_unused const struct mlx5_flow_tunnel *t,
7927                                 __rte_unused uint32_t group,
7928                                 __rte_unused uint32_t *table,
7929                                 struct rte_flow_error *error)
7930 {
7931         return rte_flow_error_set(error, ENOTSUP,
7932                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7933                                   "tunnel offload requires DV support");
7934 }
7935
7936 void
7937 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
7938                         __rte_unused  uint16_t port_id)
7939 {
7940 }
7941 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
7942