drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              struct tunnel_default_miss_ctx *ctx,
  54                              struct rte_flow_error *error);
  55 static struct mlx5_flow_tunnel *
  56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  57 static void
  58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  59 static uint32_t
  60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  61                                 const struct mlx5_flow_tunnel *tunnel,
  62                                 uint32_t group, uint32_t *table,
  63                                 struct rte_flow_error *error);
  64
  65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  66 static void mlx5_flow_pop_thread_workspace(void);
  67
  68
  69 /** Device flow drivers. */
  70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  71
  72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  73
  74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  75         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  77         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  78 #endif
  79         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  80         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  81 };
  82
  83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  85         (const int []){ \
  86                 __VA_ARGS__, 0, \
  87         }
  88
  89 /** Node object of input graph for mlx5_flow_expand_rss(). */
  90 struct mlx5_flow_expand_node {
  91         const int *const next;
  92         /**<
  93          * List of next node indexes. Index 0 is interpreted as a terminator.
  94          */
  95         const enum rte_flow_item_type type;
  96         /**< Pattern item type of current node. */
  97         uint64_t rss_types;
  98         /**<
  99          * RSS types bit-field associated with this node
 100          * (see ETH_RSS_* definitions).
 101          */
 102 };
 103
 104 /** Object returned by mlx5_flow_expand_rss(). */
 105 struct mlx5_flow_expand_rss {
 106         uint32_t entries;
 107         /**< Number of entries @p patterns and @p priorities. */
 108         struct {
 109                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 110                 uint32_t priority; /**< Priority offset for each expansion. */
 111         } entry[];
 112 };
 113
 114 static enum rte_flow_item_type
 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 116 {
 117         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 118         uint16_t ether_type = 0;
 119         uint16_t ether_type_m;
 120         uint8_t ip_next_proto = 0;
 121         uint8_t ip_next_proto_m;
 122
 123         if (item == NULL || item->spec == NULL)
 124                 return ret;
 125         switch (item->type) {
 126         case RTE_FLOW_ITEM_TYPE_ETH:
 127                 if (item->mask)
 128                         ether_type_m = ((const struct rte_flow_item_eth *)
 129                                                 (item->mask))->type;
 130                 else
 131                         ether_type_m = rte_flow_item_eth_mask.type;
 132                 if (ether_type_m != RTE_BE16(0xFFFF))
 133                         break;
 134                 ether_type = ((const struct rte_flow_item_eth *)
 135                                 (item->spec))->type;
 136                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 139                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 140                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 141                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 142                 else
 143                         ret = RTE_FLOW_ITEM_TYPE_END;
 144                 break;
 145         case RTE_FLOW_ITEM_TYPE_VLAN:
 146                 if (item->mask)
 147                         ether_type_m = ((const struct rte_flow_item_vlan *)
 148                                                 (item->mask))->inner_type;
 149                 else
 150                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 151                 if (ether_type_m != RTE_BE16(0xFFFF))
 152                         break;
 153                 ether_type = ((const struct rte_flow_item_vlan *)
 154                                 (item->spec))->inner_type;
 155                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 156                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 157                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 159                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 160                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV4:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 167                                         (item->mask))->hdr.next_proto_id;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 174                                 (item->spec))->hdr.next_proto_id;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         case RTE_FLOW_ITEM_TYPE_IPV6:
 187                 if (item->mask)
 188                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 189                                                 (item->mask))->hdr.proto;
 190                 else
 191                         ip_next_proto_m =
 192                                 rte_flow_item_ipv6_mask.hdr.proto;
 193                 if (ip_next_proto_m != 0xFF)
 194                         break;
 195                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 196                                 (item->spec))->hdr.proto;
 197                 if (ip_next_proto == IPPROTO_UDP)
 198                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 199                 else if (ip_next_proto == IPPROTO_TCP)
 200                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 201                 else if (ip_next_proto == IPPROTO_IP)
 202                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 203                 else if (ip_next_proto == IPPROTO_IPV6)
 204                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 205                 else
 206                         ret = RTE_FLOW_ITEM_TYPE_END;
 207                 break;
 208         default:
 209                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 210                 break;
 211         }
 212         return ret;
 213 }
 214
 215 /**
 216  * Expand RSS flows into several possible flows according to the RSS hash
 217  * fields requested and the driver capabilities.
 218  *
 219  * @param[out] buf
 220  *   Buffer to store the result expansion.
 221  * @param[in] size
 222  *   Buffer size in bytes. If 0, @p buf can be NULL.
 223  * @param[in] pattern
 224  *   User flow pattern.
 225  * @param[in] types
 226  *   RSS types to expand (see ETH_RSS_* definitions).
 227  * @param[in] graph
 228  *   Input graph to expand @p pattern according to @p types.
 229  * @param[in] graph_root_index
 230  *   Index of root node in @p graph, typically 0.
 231  *
 232  * @return
 233  *   A positive value representing the size of @p buf in bytes regardless of
 234  *   @p size on success, a negative errno value otherwise and rte_errno is
 235  *   set, the following errors are defined:
 236  *
 237  *   -E2BIG: graph-depth @p graph is too deep.
 238  */
 239 static int
 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 241                      const struct rte_flow_item *pattern, uint64_t types,
 242                      const struct mlx5_flow_expand_node graph[],
 243                      int graph_root_index)
 244 {
 245         const int elt_n = 8;
 246         const struct rte_flow_item *item;
 247         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 248         const int *next_node;
 249         const int *stack[elt_n];
 250         int stack_pos = 0;
 251         struct rte_flow_item flow_items[elt_n];
 252         unsigned int i;
 253         size_t lsize;
 254         size_t user_pattern_size = 0;
 255         void *addr = NULL;
 256         const struct mlx5_flow_expand_node *next = NULL;
 257         struct rte_flow_item missed_item;
 258         int missed = 0;
 259         int elt = 0;
 260         const struct rte_flow_item *last_item = NULL;
 261
 262         memset(&missed_item, 0, sizeof(missed_item));
 263         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 264                 elt_n * sizeof(buf->entry[0]);
 265         if (lsize <= size) {
 266                 buf->entry[0].priority = 0;
 267                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 268                 buf->entries = 0;
 269                 addr = buf->entry[0].pattern;
 270         }
 271         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 272                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 273                         last_item = item;
 274                 for (i = 0; node->next && node->next[i]; ++i) {
 275                         next = &graph[node->next[i]];
 276                         if (next->type == item->type)
 277                                 break;
 278                 }
 279                 if (next)
 280                         node = next;
 281                 user_pattern_size += sizeof(*item);
 282         }
 283         user_pattern_size += sizeof(*item); /* Handle END item. */
 284         lsize += user_pattern_size;
 285         /* Copy the user pattern in the first entry of the buffer. */
 286         if (lsize <= size) {
 287                 rte_memcpy(addr, pattern, user_pattern_size);
 288                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 289                 buf->entries = 1;
 290         }
 291         /* Start expanding. */
 292         memset(flow_items, 0, sizeof(flow_items));
 293         user_pattern_size -= sizeof(*item);
 294         /*
 295          * Check if the last valid item has spec set, need complete pattern,
 296          * and the pattern can be used for expansion.
 297          */
 298         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 299         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 300                 /* Item type END indicates expansion is not required. */
 301                 return lsize;
 302         }
 303         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 304                 next = NULL;
 305                 missed = 1;
 306                 for (i = 0; node->next && node->next[i]; ++i) {
 307                         next = &graph[node->next[i]];
 308                         if (next->type == missed_item.type) {
 309                                 flow_items[0].type = missed_item.type;
 310                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 311                                 break;
 312                         }
 313                         next = NULL;
 314                 }
 315         }
 316         if (next && missed) {
 317                 elt = 2; /* missed item + item end. */
 318                 node = next;
 319                 lsize += elt * sizeof(*item) + user_pattern_size;
 320                 if ((node->rss_types & types) && lsize <= size) {
 321                         buf->entry[buf->entries].priority = 1;
 322                         buf->entry[buf->entries].pattern = addr;
 323                         buf->entries++;
 324                         rte_memcpy(addr, buf->entry[0].pattern,
 325                                    user_pattern_size);
 326                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 327                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 328                         addr = (void *)(((uintptr_t)addr) +
 329                                         elt * sizeof(*item));
 330                 }
 331         }
 332         memset(flow_items, 0, sizeof(flow_items));
 333         next_node = node->next;
 334         stack[stack_pos] = next_node;
 335         node = next_node ? &graph[*next_node] : NULL;
 336         while (node) {
 337                 flow_items[stack_pos].type = node->type;
 338                 if (node->rss_types & types) {
 339                         /*
 340                          * compute the number of items to copy from the
 341                          * expansion and copy it.
 342                          * When the stack_pos is 0, there are 1 element in it,
 343                          * plus the addition END item.
 344                          */
 345                         elt = stack_pos + 2;
 346                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 347                         lsize += elt * sizeof(*item) + user_pattern_size;
 348                         if (lsize <= size) {
 349                                 size_t n = elt * sizeof(*item);
 350
 351                                 buf->entry[buf->entries].priority =
 352                                         stack_pos + 1 + missed;
 353                                 buf->entry[buf->entries].pattern = addr;
 354                                 buf->entries++;
 355                                 rte_memcpy(addr, buf->entry[0].pattern,
 356                                            user_pattern_size);
 357                                 addr = (void *)(((uintptr_t)addr) +
 358                                                 user_pattern_size);
 359                                 rte_memcpy(addr, &missed_item,
 360                                            missed * sizeof(*item));
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                         missed * sizeof(*item));
 363                                 rte_memcpy(addr, flow_items, n);
 364                                 addr = (void *)(((uintptr_t)addr) + n);
 365                         }
 366                 }
 367                 /* Go deeper. */
 368                 if (node->next) {
 369                         next_node = node->next;
 370                         if (stack_pos++ == elt_n) {
 371                                 rte_errno = E2BIG;
 372                                 return -rte_errno;
 373                         }
 374                         stack[stack_pos] = next_node;
 375                 } else if (*(next_node + 1)) {
 376                         /* Follow up with the next possibility. */
 377                         ++next_node;
 378                 } else {
 379                         /* Move to the next path. */
 380                         if (stack_pos)
 381                                 next_node = stack[--stack_pos];
 382                         next_node++;
 383                         stack[stack_pos] = next_node;
 384                 }
 385                 node = *next_node ? &graph[*next_node] : NULL;
 386         };
 387         /* no expanded flows but we have missed item, create one rule for it */
 388         if (buf->entries == 1 && missed != 0) {
 389                 elt = 2;
 390                 lsize += elt * sizeof(*item) + user_pattern_size;
 391                 if (lsize <= size) {
 392                         buf->entry[buf->entries].priority = 1;
 393                         buf->entry[buf->entries].pattern = addr;
 394                         buf->entries++;
 395                         flow_items[0].type = missed_item.type;
 396                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 397                         rte_memcpy(addr, buf->entry[0].pattern,
 398                                    user_pattern_size);
 399                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 400                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 401                 }
 402         }
 403         return lsize;
 404 }
 405
 406 enum mlx5_expansion {
 407         MLX5_EXPANSION_ROOT,
 408         MLX5_EXPANSION_ROOT_OUTER,
 409         MLX5_EXPANSION_ROOT_ETH_VLAN,
 410         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 411         MLX5_EXPANSION_OUTER_ETH,
 412         MLX5_EXPANSION_OUTER_ETH_VLAN,
 413         MLX5_EXPANSION_OUTER_VLAN,
 414         MLX5_EXPANSION_OUTER_IPV4,
 415         MLX5_EXPANSION_OUTER_IPV4_UDP,
 416         MLX5_EXPANSION_OUTER_IPV4_TCP,
 417         MLX5_EXPANSION_OUTER_IPV6,
 418         MLX5_EXPANSION_OUTER_IPV6_UDP,
 419         MLX5_EXPANSION_OUTER_IPV6_TCP,
 420         MLX5_EXPANSION_VXLAN,
 421         MLX5_EXPANSION_VXLAN_GPE,
 422         MLX5_EXPANSION_GRE,
 423         MLX5_EXPANSION_MPLS,
 424         MLX5_EXPANSION_ETH,
 425         MLX5_EXPANSION_ETH_VLAN,
 426         MLX5_EXPANSION_VLAN,
 427         MLX5_EXPANSION_IPV4,
 428         MLX5_EXPANSION_IPV4_UDP,
 429         MLX5_EXPANSION_IPV4_TCP,
 430         MLX5_EXPANSION_IPV6,
 431         MLX5_EXPANSION_IPV6_UDP,
 432         MLX5_EXPANSION_IPV6_TCP,
 433 };
 434
 435 /** Supported expansion of items. */
 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 437         [MLX5_EXPANSION_ROOT] = {
 438                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 439                                                   MLX5_EXPANSION_IPV4,
 440                                                   MLX5_EXPANSION_IPV6),
 441                 .type = RTE_FLOW_ITEM_TYPE_END,
 442         },
 443         [MLX5_EXPANSION_ROOT_OUTER] = {
 444                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 445                                                   MLX5_EXPANSION_OUTER_IPV4,
 446                                                   MLX5_EXPANSION_OUTER_IPV6),
 447                 .type = RTE_FLOW_ITEM_TYPE_END,
 448         },
 449         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 450                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 451                 .type = RTE_FLOW_ITEM_TYPE_END,
 452         },
 453         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 454                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 455                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_END,
 457         },
 458         [MLX5_EXPANSION_OUTER_ETH] = {
 459                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 460                                                   MLX5_EXPANSION_OUTER_IPV6,
 461                                                   MLX5_EXPANSION_MPLS),
 462                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 463                 .rss_types = 0,
 464         },
 465         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 466                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 467                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 468                 .rss_types = 0,
 469         },
 470         [MLX5_EXPANSION_OUTER_VLAN] = {
 471                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 472                                                   MLX5_EXPANSION_OUTER_IPV6),
 473                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 477                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 478                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 479                          MLX5_EXPANSION_GRE,
 480                          MLX5_EXPANSION_IPV4,
 481                          MLX5_EXPANSION_IPV6),
 482                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 483                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 484                         ETH_RSS_NONFRAG_IPV4_OTHER,
 485         },
 486         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 487                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 488                                                   MLX5_EXPANSION_VXLAN_GPE),
 489                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 490                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 493                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 494                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 498                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 499                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 500                          MLX5_EXPANSION_IPV4,
 501                          MLX5_EXPANSION_IPV6),
 502                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 503                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 504                         ETH_RSS_NONFRAG_IPV6_OTHER,
 505         },
 506         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 508                                                   MLX5_EXPANSION_VXLAN_GPE),
 509                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 510                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 511         },
 512         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 513                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 514                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 515         },
 516         [MLX5_EXPANSION_VXLAN] = {
 517                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 518                                                   MLX5_EXPANSION_IPV4,
 519                                                   MLX5_EXPANSION_IPV6),
 520                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 521         },
 522         [MLX5_EXPANSION_VXLAN_GPE] = {
 523                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 524                                                   MLX5_EXPANSION_IPV4,
 525                                                   MLX5_EXPANSION_IPV6),
 526                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 527         },
 528         [MLX5_EXPANSION_GRE] = {
 529                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 530                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 531         },
 532         [MLX5_EXPANSION_MPLS] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 534                                                   MLX5_EXPANSION_IPV6),
 535                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 536         },
 537         [MLX5_EXPANSION_ETH] = {
 538                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 539                                                   MLX5_EXPANSION_IPV6),
 540                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 541         },
 542         [MLX5_EXPANSION_ETH_VLAN] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 544                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 545         },
 546         [MLX5_EXPANSION_VLAN] = {
 547                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 548                                                   MLX5_EXPANSION_IPV6),
 549                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 550         },
 551         [MLX5_EXPANSION_IPV4] = {
 552                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 553                                                   MLX5_EXPANSION_IPV4_TCP),
 554                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 555                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 556                         ETH_RSS_NONFRAG_IPV4_OTHER,
 557         },
 558         [MLX5_EXPANSION_IPV4_UDP] = {
 559                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 560                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 561         },
 562         [MLX5_EXPANSION_IPV4_TCP] = {
 563                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 564                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 565         },
 566         [MLX5_EXPANSION_IPV6] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 568                                                   MLX5_EXPANSION_IPV6_TCP),
 569                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 570                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 571                         ETH_RSS_NONFRAG_IPV6_OTHER,
 572         },
 573         [MLX5_EXPANSION_IPV6_UDP] = {
 574                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 575                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 576         },
 577         [MLX5_EXPANSION_IPV6_TCP] = {
 578                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 579                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 580         },
 581 };
 582
 583 static struct rte_flow_shared_action *
 584 mlx5_shared_action_create(struct rte_eth_dev *dev,
 585                           const struct rte_flow_shared_action_conf *conf,
 586                           const struct rte_flow_action *action,
 587                           struct rte_flow_error *error);
 588 static int mlx5_shared_action_destroy
 589                                 (struct rte_eth_dev *dev,
 590                                  struct rte_flow_shared_action *shared_action,
 591                                  struct rte_flow_error *error);
 592 static int mlx5_shared_action_update
 593                                 (struct rte_eth_dev *dev,
 594                                  struct rte_flow_shared_action *shared_action,
 595                                  const struct rte_flow_action *action,
 596                                  struct rte_flow_error *error);
 597 static int mlx5_shared_action_query
 598                                 (struct rte_eth_dev *dev,
 599                                  const struct rte_flow_shared_action *action,
 600                                  void *data,
 601                                  struct rte_flow_error *error);
 602 static int
 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 604                     struct rte_flow_tunnel *app_tunnel,
 605                     struct rte_flow_action **actions,
 606                     uint32_t *num_of_actions,
 607                     struct rte_flow_error *error);
 608 static int
 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 610                        struct rte_flow_tunnel *app_tunnel,
 611                        struct rte_flow_item **items,
 612                        uint32_t *num_of_items,
 613                        struct rte_flow_error *error);
 614 static int
 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 616                               struct rte_flow_item *pmd_items,
 617                               uint32_t num_items, struct rte_flow_error *err);
 618 static int
 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 620                                 struct rte_flow_action *pmd_actions,
 621                                 uint32_t num_actions,
 622                                 struct rte_flow_error *err);
 623 static int
 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 625                                   struct rte_mbuf *m,
 626                                   struct rte_flow_restore_info *info,
 627                                   struct rte_flow_error *err);
 628
 629 static const struct rte_flow_ops mlx5_flow_ops = {
 630         .validate = mlx5_flow_validate,
 631         .create = mlx5_flow_create,
 632         .destroy = mlx5_flow_destroy,
 633         .flush = mlx5_flow_flush,
 634         .isolate = mlx5_flow_isolate,
 635         .query = mlx5_flow_query,
 636         .dev_dump = mlx5_flow_dev_dump,
 637         .get_aged_flows = mlx5_flow_get_aged_flows,
 638         .shared_action_create = mlx5_shared_action_create,
 639         .shared_action_destroy = mlx5_shared_action_destroy,
 640         .shared_action_update = mlx5_shared_action_update,
 641         .shared_action_query = mlx5_shared_action_query,
 642         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 643         .tunnel_match = mlx5_flow_tunnel_match,
 644         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 645         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 646         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 647 };
 648
 649 /* Tunnel information. */
 650 struct mlx5_flow_tunnel_info {
 651         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 652         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 653 };
 654
 655 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 656         {
 657                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 658                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 659         },
 660         {
 661                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 662                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 663         },
 664         {
 665                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 666                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 667         },
 668         {
 669                 .tunnel = MLX5_FLOW_LAYER_GRE,
 670                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 671         },
 672         {
 673                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 674                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 675         },
 676         {
 677                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 678                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 679         },
 680         {
 681                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 682                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 683         },
 684         {
 685                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 686                 .ptype = RTE_PTYPE_TUNNEL_IP,
 687         },
 688         {
 689                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 690                 .ptype = RTE_PTYPE_TUNNEL_IP,
 691         },
 692         {
 693                 .tunnel = MLX5_FLOW_LAYER_GTP,
 694                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 695         },
 696 };
 697
 698 /* Key of thread specific flow workspace data. */
 699 static pthread_key_t key_workspace;
 700
 701 /* Thread specific flow workspace data once initialization data. */
 702 static pthread_once_t key_workspace_init;
 703
 704
 705 /**
 706  * Translate tag ID to register.
 707  *
 708  * @param[in] dev
 709  *   Pointer to the Ethernet device structure.
 710  * @param[in] feature
 711  *   The feature that request the register.
 712  * @param[in] id
 713  *   The request register ID.
 714  * @param[out] error
 715  *   Error description in case of any.
 716  *
 717  * @return
 718  *   The request register on success, a negative errno
 719  *   value otherwise and rte_errno is set.
 720  */
 721 int
 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 723                      enum mlx5_feature_name feature,
 724                      uint32_t id,
 725                      struct rte_flow_error *error)
 726 {
 727         struct mlx5_priv *priv = dev->data->dev_private;
 728         struct mlx5_dev_config *config = &priv->config;
 729         enum modify_reg start_reg;
 730         bool skip_mtr_reg = false;
 731
 732         switch (feature) {
 733         case MLX5_HAIRPIN_RX:
 734                 return REG_B;
 735         case MLX5_HAIRPIN_TX:
 736                 return REG_A;
 737         case MLX5_METADATA_RX:
 738                 switch (config->dv_xmeta_en) {
 739                 case MLX5_XMETA_MODE_LEGACY:
 740                         return REG_B;
 741                 case MLX5_XMETA_MODE_META16:
 742                         return REG_C_0;
 743                 case MLX5_XMETA_MODE_META32:
 744                         return REG_C_1;
 745                 }
 746                 break;
 747         case MLX5_METADATA_TX:
 748                 return REG_A;
 749         case MLX5_METADATA_FDB:
 750                 switch (config->dv_xmeta_en) {
 751                 case MLX5_XMETA_MODE_LEGACY:
 752                         return REG_NON;
 753                 case MLX5_XMETA_MODE_META16:
 754                         return REG_C_0;
 755                 case MLX5_XMETA_MODE_META32:
 756                         return REG_C_1;
 757                 }
 758                 break;
 759         case MLX5_FLOW_MARK:
 760                 switch (config->dv_xmeta_en) {
 761                 case MLX5_XMETA_MODE_LEGACY:
 762                         return REG_NON;
 763                 case MLX5_XMETA_MODE_META16:
 764                         return REG_C_1;
 765                 case MLX5_XMETA_MODE_META32:
 766                         return REG_C_0;
 767                 }
 768                 break;
 769         case MLX5_MTR_SFX:
 770                 /*
 771                  * If meter color and flow match share one register, flow match
 772                  * should use the meter color register for match.
 773                  */
 774                 if (priv->mtr_reg_share)
 775                         return priv->mtr_color_reg;
 776                 else
 777                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 778                                REG_C_3;
 779         case MLX5_MTR_COLOR:
 780                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 781                 return priv->mtr_color_reg;
 782         case MLX5_COPY_MARK:
 783                 /*
 784                  * Metadata COPY_MARK register using is in meter suffix sub
 785                  * flow while with meter. It's safe to share the same register.
 786                  */
 787                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 788         case MLX5_APP_TAG:
 789                 /*
 790                  * If meter is enable, it will engage the register for color
 791                  * match and flow match. If meter color match is not using the
 792                  * REG_C_2, need to skip the REG_C_x be used by meter color
 793                  * match.
 794                  * If meter is disable, free to use all available registers.
 795                  */
 796                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 797                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 798                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 799                 if (id > (REG_C_7 - start_reg))
 800                         return rte_flow_error_set(error, EINVAL,
 801                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 802                                                   NULL, "invalid tag id");
 803                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 804                         return rte_flow_error_set(error, ENOTSUP,
 805                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 806                                                   NULL, "unsupported tag id");
 807                 /*
 808                  * This case means meter is using the REG_C_x great than 2.
 809                  * Take care not to conflict with meter color REG_C_x.
 810                  * If the available index REG_C_y >= REG_C_x, skip the
 811                  * color register.
 812                  */
 813                 if (skip_mtr_reg && config->flow_mreg_c
 814                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 815                         if (id >= (REG_C_7 - start_reg))
 816                                 return rte_flow_error_set(error, EINVAL,
 817                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 818                                                         NULL, "invalid tag id");
 819                         if (config->flow_mreg_c
 820                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 821                                 return config->flow_mreg_c
 822                                                [id + 1 + start_reg - REG_C_0];
 823                         return rte_flow_error_set(error, ENOTSUP,
 824                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 825                                                   NULL, "unsupported tag id");
 826                 }
 827                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 828         }
 829         MLX5_ASSERT(false);
 830         return rte_flow_error_set(error, EINVAL,
 831                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 832                                   NULL, "invalid feature name");
 833 }
 834
 835 /**
 836  * Check extensive flow metadata register support.
 837  *
 838  * @param dev
 839  *   Pointer to rte_eth_dev structure.
 840  *
 841  * @return
 842  *   True if device supports extensive flow metadata register, otherwise false.
 843  */
 844 bool
 845 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 846 {
 847         struct mlx5_priv *priv = dev->data->dev_private;
 848         struct mlx5_dev_config *config = &priv->config;
 849
 850         /*
 851          * Having available reg_c can be regarded inclusively as supporting
 852          * extensive flow metadata register, which could mean,
 853          * - metadata register copy action by modify header.
 854          * - 16 modify header actions is supported.
 855          * - reg_c's are preserved across different domain (FDB and NIC) on
 856          *   packet loopback by flow lookup miss.
 857          */
 858         return config->flow_mreg_c[2] != REG_NON;
 859 }
 860
 861 /**
 862  * Verify the @p item specifications (spec, last, mask) are compatible with the
 863  * NIC capabilities.
 864  *
 865  * @param[in] item
 866  *   Item specification.
 867  * @param[in] mask
 868  *   @p item->mask or flow default bit-masks.
 869  * @param[in] nic_mask
 870  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 871  * @param[in] size
 872  *   Bit-masks size in bytes.
 873  * @param[in] range_accepted
 874  *   True if range of values is accepted for specific fields, false otherwise.
 875  * @param[out] error
 876  *   Pointer to error structure.
 877  *
 878  * @return
 879  *   0 on success, a negative errno value otherwise and rte_errno is set.
 880  */
 881 int
 882 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 883                           const uint8_t *mask,
 884                           const uint8_t *nic_mask,
 885                           unsigned int size,
 886                           bool range_accepted,
 887                           struct rte_flow_error *error)
 888 {
 889         unsigned int i;
 890
 891         MLX5_ASSERT(nic_mask);
 892         for (i = 0; i < size; ++i)
 893                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 894                         return rte_flow_error_set(error, ENOTSUP,
 895                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 896                                                   item,
 897                                                   "mask enables non supported"
 898                                                   " bits");
 899         if (!item->spec && (item->mask || item->last))
 900                 return rte_flow_error_set(error, EINVAL,
 901                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 902                                           "mask/last without a spec is not"
 903                                           " supported");
 904         if (item->spec && item->last && !range_accepted) {
 905                 uint8_t spec[size];
 906                 uint8_t last[size];
 907                 unsigned int i;
 908                 int ret;
 909
 910                 for (i = 0; i < size; ++i) {
 911                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 912                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 913                 }
 914                 ret = memcmp(spec, last, size);
 915                 if (ret != 0)
 916                         return rte_flow_error_set(error, EINVAL,
 917                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 918                                                   item,
 919                                                   "range is not valid");
 920         }
 921         return 0;
 922 }
 923
 924 /**
 925  * Adjust the hash fields according to the @p flow information.
 926  *
 927  * @param[in] dev_flow.
 928  *   Pointer to the mlx5_flow.
 929  * @param[in] tunnel
 930  *   1 when the hash field is for a tunnel item.
 931  * @param[in] layer_types
 932  *   ETH_RSS_* types.
 933  * @param[in] hash_fields
 934  *   Item hash fields.
 935  *
 936  * @return
 937  *   The hash fields that should be used.
 938  */
 939 uint64_t
 940 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 941                             int tunnel __rte_unused, uint64_t layer_types,
 942                             uint64_t hash_fields)
 943 {
 944 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 945         int rss_request_inner = rss_desc->level >= 2;
 946
 947         /* Check RSS hash level for tunnel. */
 948         if (tunnel && rss_request_inner)
 949                 hash_fields |= IBV_RX_HASH_INNER;
 950         else if (tunnel || rss_request_inner)
 951                 return 0;
 952 #endif
 953         /* Check if requested layer matches RSS hash fields. */
 954         if (!(rss_desc->types & layer_types))
 955                 return 0;
 956         return hash_fields;
 957 }
 958
 959 /**
 960  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 961  * if several tunnel rules are used on this queue, the tunnel ptype will be
 962  * cleared.
 963  *
 964  * @param rxq_ctrl
 965  *   Rx queue to update.
 966  */
 967 static void
 968 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 969 {
 970         unsigned int i;
 971         uint32_t tunnel_ptype = 0;
 972
 973         /* Look up for the ptype to use. */
 974         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 975                 if (!rxq_ctrl->flow_tunnels_n[i])
 976                         continue;
 977                 if (!tunnel_ptype) {
 978                         tunnel_ptype = tunnels_info[i].ptype;
 979                 } else {
 980                         tunnel_ptype = 0;
 981                         break;
 982                 }
 983         }
 984         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 985 }
 986
 987 /**
 988  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 989  * flow.
 990  *
 991  * @param[in] dev
 992  *   Pointer to the Ethernet device structure.
 993  * @param[in] dev_handle
 994  *   Pointer to device flow handle structure.
 995  */
 996 static void
 997 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
 998                        struct mlx5_flow_handle *dev_handle)
 999 {
1000         struct mlx5_priv *priv = dev->data->dev_private;
1001         const int mark = dev_handle->mark;
1002         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1003         struct mlx5_hrxq *hrxq;
1004         unsigned int i;
1005
1006         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1007                 return;
1008         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1009                               dev_handle->rix_hrxq);
1010         if (!hrxq)
1011                 return;
1012         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1013                 int idx = hrxq->ind_table->queues[i];
1014                 struct mlx5_rxq_ctrl *rxq_ctrl =
1015                         container_of((*priv->rxqs)[idx],
1016                                      struct mlx5_rxq_ctrl, rxq);
1017
1018                 /*
1019                  * To support metadata register copy on Tx loopback,
1020                  * this must be always enabled (metadata may arive
1021                  * from other port - not from local flows only.
1022                  */
1023                 if (priv->config.dv_flow_en &&
1024                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1025                     mlx5_flow_ext_mreg_supported(dev)) {
1026                         rxq_ctrl->rxq.mark = 1;
1027                         rxq_ctrl->flow_mark_n = 1;
1028                 } else if (mark) {
1029                         rxq_ctrl->rxq.mark = 1;
1030                         rxq_ctrl->flow_mark_n++;
1031                 }
1032                 if (tunnel) {
1033                         unsigned int j;
1034
1035                         /* Increase the counter matching the flow. */
1036                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1037                                 if ((tunnels_info[j].tunnel &
1038                                      dev_handle->layers) ==
1039                                     tunnels_info[j].tunnel) {
1040                                         rxq_ctrl->flow_tunnels_n[j]++;
1041                                         break;
1042                                 }
1043                         }
1044                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1045                 }
1046         }
1047 }
1048
1049 /**
1050  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1051  *
1052  * @param[in] dev
1053  *   Pointer to the Ethernet device structure.
1054  * @param[in] flow
1055  *   Pointer to flow structure.
1056  */
1057 static void
1058 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1059 {
1060         struct mlx5_priv *priv = dev->data->dev_private;
1061         uint32_t handle_idx;
1062         struct mlx5_flow_handle *dev_handle;
1063
1064         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1065                        handle_idx, dev_handle, next)
1066                 flow_drv_rxq_flags_set(dev, dev_handle);
1067 }
1068
1069 /**
1070  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1071  * device flow if no other flow uses it with the same kind of request.
1072  *
1073  * @param dev
1074  *   Pointer to Ethernet device.
1075  * @param[in] dev_handle
1076  *   Pointer to the device flow handle structure.
1077  */
1078 static void
1079 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1080                         struct mlx5_flow_handle *dev_handle)
1081 {
1082         struct mlx5_priv *priv = dev->data->dev_private;
1083         const int mark = dev_handle->mark;
1084         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1085         struct mlx5_hrxq *hrxq;
1086         unsigned int i;
1087
1088         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1089                 return;
1090         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1091                               dev_handle->rix_hrxq);
1092         if (!hrxq)
1093                 return;
1094         MLX5_ASSERT(dev->data->dev_started);
1095         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1096                 int idx = hrxq->ind_table->queues[i];
1097                 struct mlx5_rxq_ctrl *rxq_ctrl =
1098                         container_of((*priv->rxqs)[idx],
1099                                      struct mlx5_rxq_ctrl, rxq);
1100
1101                 if (priv->config.dv_flow_en &&
1102                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1103                     mlx5_flow_ext_mreg_supported(dev)) {
1104                         rxq_ctrl->rxq.mark = 1;
1105                         rxq_ctrl->flow_mark_n = 1;
1106                 } else if (mark) {
1107                         rxq_ctrl->flow_mark_n--;
1108                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1109                 }
1110                 if (tunnel) {
1111                         unsigned int j;
1112
1113                         /* Decrease the counter matching the flow. */
1114                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1115                                 if ((tunnels_info[j].tunnel &
1116                                      dev_handle->layers) ==
1117                                     tunnels_info[j].tunnel) {
1118                                         rxq_ctrl->flow_tunnels_n[j]--;
1119                                         break;
1120                                 }
1121                         }
1122                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1123                 }
1124         }
1125 }
1126
1127 /**
1128  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1129  * @p flow if no other flow uses it with the same kind of request.
1130  *
1131  * @param dev
1132  *   Pointer to Ethernet device.
1133  * @param[in] flow
1134  *   Pointer to the flow.
1135  */
1136 static void
1137 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1138 {
1139         struct mlx5_priv *priv = dev->data->dev_private;
1140         uint32_t handle_idx;
1141         struct mlx5_flow_handle *dev_handle;
1142
1143         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1144                        handle_idx, dev_handle, next)
1145                 flow_drv_rxq_flags_trim(dev, dev_handle);
1146 }
1147
1148 /**
1149  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1150  *
1151  * @param dev
1152  *   Pointer to Ethernet device.
1153  */
1154 static void
1155 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1156 {
1157         struct mlx5_priv *priv = dev->data->dev_private;
1158         unsigned int i;
1159
1160         for (i = 0; i != priv->rxqs_n; ++i) {
1161                 struct mlx5_rxq_ctrl *rxq_ctrl;
1162                 unsigned int j;
1163
1164                 if (!(*priv->rxqs)[i])
1165                         continue;
1166                 rxq_ctrl = container_of((*priv->rxqs)[i],
1167                                         struct mlx5_rxq_ctrl, rxq);
1168                 rxq_ctrl->flow_mark_n = 0;
1169                 rxq_ctrl->rxq.mark = 0;
1170                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1171                         rxq_ctrl->flow_tunnels_n[j] = 0;
1172                 rxq_ctrl->rxq.tunnel = 0;
1173         }
1174 }
1175
1176 /**
1177  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1178  *
1179  * @param[in] dev
1180  *   Pointer to the Ethernet device structure.
1181  */
1182 void
1183 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1184 {
1185         struct mlx5_priv *priv = dev->data->dev_private;
1186         struct mlx5_rxq_data *data;
1187         unsigned int i;
1188
1189         for (i = 0; i != priv->rxqs_n; ++i) {
1190                 if (!(*priv->rxqs)[i])
1191                         continue;
1192                 data = (*priv->rxqs)[i];
1193                 if (!rte_flow_dynf_metadata_avail()) {
1194                         data->dynf_meta = 0;
1195                         data->flow_meta_mask = 0;
1196                         data->flow_meta_offset = -1;
1197                 } else {
1198                         data->dynf_meta = 1;
1199                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1200                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1201                 }
1202         }
1203 }
1204
1205 /*
1206  * return a pointer to the desired action in the list of actions.
1207  *
1208  * @param[in] actions
1209  *   The list of actions to search the action in.
1210  * @param[in] action
1211  *   The action to find.
1212  *
1213  * @return
1214  *   Pointer to the action in the list, if found. NULL otherwise.
1215  */
1216 const struct rte_flow_action *
1217 mlx5_flow_find_action(const struct rte_flow_action *actions,
1218                       enum rte_flow_action_type action)
1219 {
1220         if (actions == NULL)
1221                 return NULL;
1222         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1223                 if (actions->type == action)
1224                         return actions;
1225         return NULL;
1226 }
1227
1228 /*
1229  * Validate the flag action.
1230  *
1231  * @param[in] action_flags
1232  *   Bit-fields that holds the actions detected until now.
1233  * @param[in] attr
1234  *   Attributes of flow that includes this action.
1235  * @param[out] error
1236  *   Pointer to error structure.
1237  *
1238  * @return
1239  *   0 on success, a negative errno value otherwise and rte_errno is set.
1240  */
1241 int
1242 mlx5_flow_validate_action_flag(uint64_t action_flags,
1243                                const struct rte_flow_attr *attr,
1244                                struct rte_flow_error *error)
1245 {
1246         if (action_flags & MLX5_FLOW_ACTION_MARK)
1247                 return rte_flow_error_set(error, EINVAL,
1248                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1249                                           "can't mark and flag in same flow");
1250         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1251                 return rte_flow_error_set(error, EINVAL,
1252                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1253                                           "can't have 2 flag"
1254                                           " actions in same flow");
1255         if (attr->egress)
1256                 return rte_flow_error_set(error, ENOTSUP,
1257                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1258                                           "flag action not supported for "
1259                                           "egress");
1260         return 0;
1261 }
1262
1263 /*
1264  * Validate the mark action.
1265  *
1266  * @param[in] action
1267  *   Pointer to the queue action.
1268  * @param[in] action_flags
1269  *   Bit-fields that holds the actions detected until now.
1270  * @param[in] attr
1271  *   Attributes of flow that includes this action.
1272  * @param[out] error
1273  *   Pointer to error structure.
1274  *
1275  * @return
1276  *   0 on success, a negative errno value otherwise and rte_errno is set.
1277  */
1278 int
1279 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1280                                uint64_t action_flags,
1281                                const struct rte_flow_attr *attr,
1282                                struct rte_flow_error *error)
1283 {
1284         const struct rte_flow_action_mark *mark = action->conf;
1285
1286         if (!mark)
1287                 return rte_flow_error_set(error, EINVAL,
1288                                           RTE_FLOW_ERROR_TYPE_ACTION,
1289                                           action,
1290                                           "configuration cannot be null");
1291         if (mark->id >= MLX5_FLOW_MARK_MAX)
1292                 return rte_flow_error_set(error, EINVAL,
1293                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1294                                           &mark->id,
1295                                           "mark id must in 0 <= id < "
1296                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1297         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1298                 return rte_flow_error_set(error, EINVAL,
1299                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1300                                           "can't flag and mark in same flow");
1301         if (action_flags & MLX5_FLOW_ACTION_MARK)
1302                 return rte_flow_error_set(error, EINVAL,
1303                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1304                                           "can't have 2 mark actions in same"
1305                                           " flow");
1306         if (attr->egress)
1307                 return rte_flow_error_set(error, ENOTSUP,
1308                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1309                                           "mark action not supported for "
1310                                           "egress");
1311         return 0;
1312 }
1313
1314 /*
1315  * Validate the drop action.
1316  *
1317  * @param[in] action_flags
1318  *   Bit-fields that holds the actions detected until now.
1319  * @param[in] attr
1320  *   Attributes of flow that includes this action.
1321  * @param[out] error
1322  *   Pointer to error structure.
1323  *
1324  * @return
1325  *   0 on success, a negative errno value otherwise and rte_errno is set.
1326  */
1327 int
1328 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1329                                const struct rte_flow_attr *attr,
1330                                struct rte_flow_error *error)
1331 {
1332         if (attr->egress)
1333                 return rte_flow_error_set(error, ENOTSUP,
1334                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1335                                           "drop action not supported for "
1336                                           "egress");
1337         return 0;
1338 }
1339
1340 /*
1341  * Validate the queue action.
1342  *
1343  * @param[in] action
1344  *   Pointer to the queue action.
1345  * @param[in] action_flags
1346  *   Bit-fields that holds the actions detected until now.
1347  * @param[in] dev
1348  *   Pointer to the Ethernet device structure.
1349  * @param[in] attr
1350  *   Attributes of flow that includes this action.
1351  * @param[out] error
1352  *   Pointer to error structure.
1353  *
1354  * @return
1355  *   0 on success, a negative errno value otherwise and rte_errno is set.
1356  */
1357 int
1358 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1359                                 uint64_t action_flags,
1360                                 struct rte_eth_dev *dev,
1361                                 const struct rte_flow_attr *attr,
1362                                 struct rte_flow_error *error)
1363 {
1364         struct mlx5_priv *priv = dev->data->dev_private;
1365         const struct rte_flow_action_queue *queue = action->conf;
1366
1367         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1368                 return rte_flow_error_set(error, EINVAL,
1369                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1370                                           "can't have 2 fate actions in"
1371                                           " same flow");
1372         if (!priv->rxqs_n)
1373                 return rte_flow_error_set(error, EINVAL,
1374                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1375                                           NULL, "No Rx queues configured");
1376         if (queue->index >= priv->rxqs_n)
1377                 return rte_flow_error_set(error, EINVAL,
1378                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1379                                           &queue->index,
1380                                           "queue index out of range");
1381         if (!(*priv->rxqs)[queue->index])
1382                 return rte_flow_error_set(error, EINVAL,
1383                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1384                                           &queue->index,
1385                                           "queue is not configured");
1386         if (attr->egress)
1387                 return rte_flow_error_set(error, ENOTSUP,
1388                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1389                                           "queue action not supported for "
1390                                           "egress");
1391         return 0;
1392 }
1393
1394 /*
1395  * Validate the rss action.
1396  *
1397  * @param[in] dev
1398  *   Pointer to the Ethernet device structure.
1399  * @param[in] action
1400  *   Pointer to the queue action.
1401  * @param[out] error
1402  *   Pointer to error structure.
1403  *
1404  * @return
1405  *   0 on success, a negative errno value otherwise and rte_errno is set.
1406  */
1407 int
1408 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1409                          const struct rte_flow_action *action,
1410                          struct rte_flow_error *error)
1411 {
1412         struct mlx5_priv *priv = dev->data->dev_private;
1413         const struct rte_flow_action_rss *rss = action->conf;
1414         unsigned int i;
1415
1416         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1417             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1418                 return rte_flow_error_set(error, ENOTSUP,
1419                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1420                                           &rss->func,
1421                                           "RSS hash function not supported");
1422 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1423         if (rss->level > 2)
1424 #else
1425         if (rss->level > 1)
1426 #endif
1427                 return rte_flow_error_set(error, ENOTSUP,
1428                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1429                                           &rss->level,
1430                                           "tunnel RSS is not supported");
1431         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1432         if (rss->key_len == 0 && rss->key != NULL)
1433                 return rte_flow_error_set(error, ENOTSUP,
1434                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1435                                           &rss->key_len,
1436                                           "RSS hash key length 0");
1437         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1438                 return rte_flow_error_set(error, ENOTSUP,
1439                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1440                                           &rss->key_len,
1441                                           "RSS hash key too small");
1442         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1443                 return rte_flow_error_set(error, ENOTSUP,
1444                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1445                                           &rss->key_len,
1446                                           "RSS hash key too large");
1447         if (rss->queue_num > priv->config.ind_table_max_size)
1448                 return rte_flow_error_set(error, ENOTSUP,
1449                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1450                                           &rss->queue_num,
1451                                           "number of queues too large");
1452         if (rss->types & MLX5_RSS_HF_MASK)
1453                 return rte_flow_error_set(error, ENOTSUP,
1454                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1455                                           &rss->types,
1456                                           "some RSS protocols are not"
1457                                           " supported");
1458         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1459             !(rss->types & ETH_RSS_IP))
1460                 return rte_flow_error_set(error, EINVAL,
1461                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1462                                           "L3 partial RSS requested but L3 RSS"
1463                                           " type not specified");
1464         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1465             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1466                 return rte_flow_error_set(error, EINVAL,
1467                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1468                                           "L4 partial RSS requested but L4 RSS"
1469                                           " type not specified");
1470         if (!priv->rxqs_n)
1471                 return rte_flow_error_set(error, EINVAL,
1472                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1473                                           NULL, "No Rx queues configured");
1474         if (!rss->queue_num)
1475                 return rte_flow_error_set(error, EINVAL,
1476                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1477                                           NULL, "No queues configured");
1478         for (i = 0; i != rss->queue_num; ++i) {
1479                 if (rss->queue[i] >= priv->rxqs_n)
1480                         return rte_flow_error_set
1481                                 (error, EINVAL,
1482                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1483                                  &rss->queue[i], "queue index out of range");
1484                 if (!(*priv->rxqs)[rss->queue[i]])
1485                         return rte_flow_error_set
1486                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1487                                  &rss->queue[i], "queue is not configured");
1488         }
1489         return 0;
1490 }
1491
1492 /*
1493  * Validate the rss action.
1494  *
1495  * @param[in] action
1496  *   Pointer to the queue action.
1497  * @param[in] action_flags
1498  *   Bit-fields that holds the actions detected until now.
1499  * @param[in] dev
1500  *   Pointer to the Ethernet device structure.
1501  * @param[in] attr
1502  *   Attributes of flow that includes this action.
1503  * @param[in] item_flags
1504  *   Items that were detected.
1505  * @param[out] error
1506  *   Pointer to error structure.
1507  *
1508  * @return
1509  *   0 on success, a negative errno value otherwise and rte_errno is set.
1510  */
1511 int
1512 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1513                               uint64_t action_flags,
1514                               struct rte_eth_dev *dev,
1515                               const struct rte_flow_attr *attr,
1516                               uint64_t item_flags,
1517                               struct rte_flow_error *error)
1518 {
1519         const struct rte_flow_action_rss *rss = action->conf;
1520         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1521         int ret;
1522
1523         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1524                 return rte_flow_error_set(error, EINVAL,
1525                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1526                                           "can't have 2 fate actions"
1527                                           " in same flow");
1528         ret = mlx5_validate_action_rss(dev, action, error);
1529         if (ret)
1530                 return ret;
1531         if (attr->egress)
1532                 return rte_flow_error_set(error, ENOTSUP,
1533                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1534                                           "rss action not supported for "
1535                                           "egress");
1536         if (rss->level > 1 && !tunnel)
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1539                                           "inner RSS is not supported for "
1540                                           "non-tunnel flows");
1541         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1542             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1543                 return rte_flow_error_set(error, EINVAL,
1544                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1545                                           "RSS on eCPRI is not supported now");
1546         }
1547         return 0;
1548 }
1549
1550 /*
1551  * Validate the default miss action.
1552  *
1553  * @param[in] action_flags
1554  *   Bit-fields that holds the actions detected until now.
1555  * @param[out] error
1556  *   Pointer to error structure.
1557  *
1558  * @return
1559  *   0 on success, a negative errno value otherwise and rte_errno is set.
1560  */
1561 int
1562 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1563                                 const struct rte_flow_attr *attr,
1564                                 struct rte_flow_error *error)
1565 {
1566         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1567                 return rte_flow_error_set(error, EINVAL,
1568                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1569                                           "can't have 2 fate actions in"
1570                                           " same flow");
1571         if (attr->egress)
1572                 return rte_flow_error_set(error, ENOTSUP,
1573                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1574                                           "default miss action not supported "
1575                                           "for egress");
1576         if (attr->group)
1577                 return rte_flow_error_set(error, ENOTSUP,
1578                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1579                                           "only group 0 is supported");
1580         if (attr->transfer)
1581                 return rte_flow_error_set(error, ENOTSUP,
1582                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1583                                           NULL, "transfer is not supported");
1584         return 0;
1585 }
1586
1587 /*
1588  * Validate the count action.
1589  *
1590  * @param[in] dev
1591  *   Pointer to the Ethernet device structure.
1592  * @param[in] attr
1593  *   Attributes of flow that includes this action.
1594  * @param[out] error
1595  *   Pointer to error structure.
1596  *
1597  * @return
1598  *   0 on success, a negative errno value otherwise and rte_errno is set.
1599  */
1600 int
1601 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1602                                 const struct rte_flow_attr *attr,
1603                                 struct rte_flow_error *error)
1604 {
1605         if (attr->egress)
1606                 return rte_flow_error_set(error, ENOTSUP,
1607                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1608                                           "count action not supported for "
1609                                           "egress");
1610         return 0;
1611 }
1612
1613 /**
1614  * Verify the @p attributes will be correctly understood by the NIC and store
1615  * them in the @p flow if everything is correct.
1616  *
1617  * @param[in] dev
1618  *   Pointer to the Ethernet device structure.
1619  * @param[in] attributes
1620  *   Pointer to flow attributes
1621  * @param[out] error
1622  *   Pointer to error structure.
1623  *
1624  * @return
1625  *   0 on success, a negative errno value otherwise and rte_errno is set.
1626  */
1627 int
1628 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1629                               const struct rte_flow_attr *attributes,
1630                               struct rte_flow_error *error)
1631 {
1632         struct mlx5_priv *priv = dev->data->dev_private;
1633         uint32_t priority_max = priv->config.flow_prio - 1;
1634
1635         if (attributes->group)
1636                 return rte_flow_error_set(error, ENOTSUP,
1637                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1638                                           NULL, "groups is not supported");
1639         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1640             attributes->priority >= priority_max)
1641                 return rte_flow_error_set(error, ENOTSUP,
1642                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1643                                           NULL, "priority out of range");
1644         if (attributes->egress)
1645                 return rte_flow_error_set(error, ENOTSUP,
1646                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1647                                           "egress is not supported");
1648         if (attributes->transfer && !priv->config.dv_esw_en)
1649                 return rte_flow_error_set(error, ENOTSUP,
1650                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1651                                           NULL, "transfer is not supported");
1652         if (!attributes->ingress)
1653                 return rte_flow_error_set(error, EINVAL,
1654                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1655                                           NULL,
1656                                           "ingress attribute is mandatory");
1657         return 0;
1658 }
1659
1660 /**
1661  * Validate ICMP6 item.
1662  *
1663  * @param[in] item
1664  *   Item specification.
1665  * @param[in] item_flags
1666  *   Bit-fields that holds the items detected until now.
1667  * @param[in] ext_vlan_sup
1668  *   Whether extended VLAN features are supported or not.
1669  * @param[out] error
1670  *   Pointer to error structure.
1671  *
1672  * @return
1673  *   0 on success, a negative errno value otherwise and rte_errno is set.
1674  */
1675 int
1676 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1677                                uint64_t item_flags,
1678                                uint8_t target_protocol,
1679                                struct rte_flow_error *error)
1680 {
1681         const struct rte_flow_item_icmp6 *mask = item->mask;
1682         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1683         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1684                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1685         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1686                                       MLX5_FLOW_LAYER_OUTER_L4;
1687         int ret;
1688
1689         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "protocol filtering not compatible"
1693                                           " with ICMP6 layer");
1694         if (!(item_flags & l3m))
1695                 return rte_flow_error_set(error, EINVAL,
1696                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1697                                           "IPv6 is mandatory to filter on"
1698                                           " ICMP6");
1699         if (item_flags & l4m)
1700                 return rte_flow_error_set(error, EINVAL,
1701                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1702                                           "multiple L4 layers not supported");
1703         if (!mask)
1704                 mask = &rte_flow_item_icmp6_mask;
1705         ret = mlx5_flow_item_acceptable
1706                 (item, (const uint8_t *)mask,
1707                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1708                  sizeof(struct rte_flow_item_icmp6),
1709                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1710         if (ret < 0)
1711                 return ret;
1712         return 0;
1713 }
1714
1715 /**
1716  * Validate ICMP item.
1717  *
1718  * @param[in] item
1719  *   Item specification.
1720  * @param[in] item_flags
1721  *   Bit-fields that holds the items detected until now.
1722  * @param[out] error
1723  *   Pointer to error structure.
1724  *
1725  * @return
1726  *   0 on success, a negative errno value otherwise and rte_errno is set.
1727  */
1728 int
1729 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1730                              uint64_t item_flags,
1731                              uint8_t target_protocol,
1732                              struct rte_flow_error *error)
1733 {
1734         const struct rte_flow_item_icmp *mask = item->mask;
1735         const struct rte_flow_item_icmp nic_mask = {
1736                 .hdr.icmp_type = 0xff,
1737                 .hdr.icmp_code = 0xff,
1738                 .hdr.icmp_ident = RTE_BE16(0xffff),
1739                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1740         };
1741         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1742         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1743                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1744         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1745                                       MLX5_FLOW_LAYER_OUTER_L4;
1746         int ret;
1747
1748         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1749                 return rte_flow_error_set(error, EINVAL,
1750                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1751                                           "protocol filtering not compatible"
1752                                           " with ICMP layer");
1753         if (!(item_flags & l3m))
1754                 return rte_flow_error_set(error, EINVAL,
1755                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1756                                           "IPv4 is mandatory to filter"
1757                                           " on ICMP");
1758         if (item_flags & l4m)
1759                 return rte_flow_error_set(error, EINVAL,
1760                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1761                                           "multiple L4 layers not supported");
1762         if (!mask)
1763                 mask = &nic_mask;
1764         ret = mlx5_flow_item_acceptable
1765                 (item, (const uint8_t *)mask,
1766                  (const uint8_t *)&nic_mask,
1767                  sizeof(struct rte_flow_item_icmp),
1768                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1769         if (ret < 0)
1770                 return ret;
1771         return 0;
1772 }
1773
1774 /**
1775  * Validate Ethernet item.
1776  *
1777  * @param[in] item
1778  *   Item specification.
1779  * @param[in] item_flags
1780  *   Bit-fields that holds the items detected until now.
1781  * @param[out] error
1782  *   Pointer to error structure.
1783  *
1784  * @return
1785  *   0 on success, a negative errno value otherwise and rte_errno is set.
1786  */
1787 int
1788 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1789                             uint64_t item_flags, bool ext_vlan_sup,
1790                             struct rte_flow_error *error)
1791 {
1792         const struct rte_flow_item_eth *mask = item->mask;
1793         const struct rte_flow_item_eth nic_mask = {
1794                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1795                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1796                 .type = RTE_BE16(0xffff),
1797                 .has_vlan = ext_vlan_sup ? 1 : 0,
1798         };
1799         int ret;
1800         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1801         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1802                                        MLX5_FLOW_LAYER_OUTER_L2;
1803
1804         if (item_flags & ethm)
1805                 return rte_flow_error_set(error, ENOTSUP,
1806                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1807                                           "multiple L2 layers not supported");
1808         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1809             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1810                 return rte_flow_error_set(error, EINVAL,
1811                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1812                                           "L2 layer should not follow "
1813                                           "L3 layers");
1814         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1815             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1816                 return rte_flow_error_set(error, EINVAL,
1817                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1818                                           "L2 layer should not follow VLAN");
1819         if (!mask)
1820                 mask = &rte_flow_item_eth_mask;
1821         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1822                                         (const uint8_t *)&nic_mask,
1823                                         sizeof(struct rte_flow_item_eth),
1824                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1825         return ret;
1826 }
1827
1828 /**
1829  * Validate VLAN item.
1830  *
1831  * @param[in] item
1832  *   Item specification.
1833  * @param[in] item_flags
1834  *   Bit-fields that holds the items detected until now.
1835  * @param[in] dev
1836  *   Ethernet device flow is being created on.
1837  * @param[out] error
1838  *   Pointer to error structure.
1839  *
1840  * @return
1841  *   0 on success, a negative errno value otherwise and rte_errno is set.
1842  */
1843 int
1844 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1845                              uint64_t item_flags,
1846                              struct rte_eth_dev *dev,
1847                              struct rte_flow_error *error)
1848 {
1849         const struct rte_flow_item_vlan *spec = item->spec;
1850         const struct rte_flow_item_vlan *mask = item->mask;
1851         const struct rte_flow_item_vlan nic_mask = {
1852                 .tci = RTE_BE16(UINT16_MAX),
1853                 .inner_type = RTE_BE16(UINT16_MAX),
1854         };
1855         uint16_t vlan_tag = 0;
1856         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1857         int ret;
1858         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1859                                         MLX5_FLOW_LAYER_INNER_L4) :
1860                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1861                                         MLX5_FLOW_LAYER_OUTER_L4);
1862         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1863                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1864
1865         if (item_flags & vlanm)
1866                 return rte_flow_error_set(error, EINVAL,
1867                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1868                                           "multiple VLAN layers not supported");
1869         else if ((item_flags & l34m) != 0)
1870                 return rte_flow_error_set(error, EINVAL,
1871                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1872                                           "VLAN cannot follow L3/L4 layer");
1873         if (!mask)
1874                 mask = &rte_flow_item_vlan_mask;
1875         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1876                                         (const uint8_t *)&nic_mask,
1877                                         sizeof(struct rte_flow_item_vlan),
1878                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1879         if (ret)
1880                 return ret;
1881         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1882                 struct mlx5_priv *priv = dev->data->dev_private;
1883
1884                 if (priv->vmwa_context) {
1885                         /*
1886                          * Non-NULL context means we have a virtual machine
1887                          * and SR-IOV enabled, we have to create VLAN interface
1888                          * to make hypervisor to setup E-Switch vport
1889                          * context correctly. We avoid creating the multiple
1890                          * VLAN interfaces, so we cannot support VLAN tag mask.
1891                          */
1892                         return rte_flow_error_set(error, EINVAL,
1893                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1894                                                   item,
1895                                                   "VLAN tag mask is not"
1896                                                   " supported in virtual"
1897                                                   " environment");
1898                 }
1899         }
1900         if (spec) {
1901                 vlan_tag = spec->tci;
1902                 vlan_tag &= mask->tci;
1903         }
1904         /*
1905          * From verbs perspective an empty VLAN is equivalent
1906          * to a packet without VLAN layer.
1907          */
1908         if (!vlan_tag)
1909                 return rte_flow_error_set(error, EINVAL,
1910                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1911                                           item->spec,
1912                                           "VLAN cannot be empty");
1913         return 0;
1914 }
1915
1916 /**
1917  * Validate IPV4 item.
1918  *
1919  * @param[in] item
1920  *   Item specification.
1921  * @param[in] item_flags
1922  *   Bit-fields that holds the items detected until now.
1923  * @param[in] last_item
1924  *   Previous validated item in the pattern items.
1925  * @param[in] ether_type
1926  *   Type in the ethernet layer header (including dot1q).
1927  * @param[in] acc_mask
1928  *   Acceptable mask, if NULL default internal default mask
1929  *   will be used to check whether item fields are supported.
1930  * @param[in] range_accepted
1931  *   True if range of values is accepted for specific fields, false otherwise.
1932  * @param[out] error
1933  *   Pointer to error structure.
1934  *
1935  * @return
1936  *   0 on success, a negative errno value otherwise and rte_errno is set.
1937  */
1938 int
1939 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1940                              uint64_t item_flags,
1941                              uint64_t last_item,
1942                              uint16_t ether_type,
1943                              const struct rte_flow_item_ipv4 *acc_mask,
1944                              bool range_accepted,
1945                              struct rte_flow_error *error)
1946 {
1947         const struct rte_flow_item_ipv4 *mask = item->mask;
1948         const struct rte_flow_item_ipv4 *spec = item->spec;
1949         const struct rte_flow_item_ipv4 nic_mask = {
1950                 .hdr = {
1951                         .src_addr = RTE_BE32(0xffffffff),
1952                         .dst_addr = RTE_BE32(0xffffffff),
1953                         .type_of_service = 0xff,
1954                         .next_proto_id = 0xff,
1955                 },
1956         };
1957         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1958         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1959                                       MLX5_FLOW_LAYER_OUTER_L3;
1960         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1961                                       MLX5_FLOW_LAYER_OUTER_L4;
1962         int ret;
1963         uint8_t next_proto = 0xFF;
1964         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1965                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1966                                   MLX5_FLOW_LAYER_INNER_VLAN);
1967
1968         if ((last_item & l2_vlan) && ether_type &&
1969             ether_type != RTE_ETHER_TYPE_IPV4)
1970                 return rte_flow_error_set(error, EINVAL,
1971                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1972                                           "IPv4 cannot follow L2/VLAN layer "
1973                                           "which ether type is not IPv4");
1974         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1975                 if (mask && spec)
1976                         next_proto = mask->hdr.next_proto_id &
1977                                      spec->hdr.next_proto_id;
1978                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1979                         return rte_flow_error_set(error, EINVAL,
1980                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1981                                                   item,
1982                                                   "multiple tunnel "
1983                                                   "not supported");
1984         }
1985         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1986                 return rte_flow_error_set(error, EINVAL,
1987                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1988                                           "wrong tunnel type - IPv6 specified "
1989                                           "but IPv4 item provided");
1990         if (item_flags & l3m)
1991                 return rte_flow_error_set(error, ENOTSUP,
1992                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1993                                           "multiple L3 layers not supported");
1994         else if (item_flags & l4m)
1995                 return rte_flow_error_set(error, EINVAL,
1996                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1997                                           "L3 cannot follow an L4 layer.");
1998         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1999                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2000                 return rte_flow_error_set(error, EINVAL,
2001                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2002                                           "L3 cannot follow an NVGRE layer.");
2003         if (!mask)
2004                 mask = &rte_flow_item_ipv4_mask;
2005         else if (mask->hdr.next_proto_id != 0 &&
2006                  mask->hdr.next_proto_id != 0xff)
2007                 return rte_flow_error_set(error, EINVAL,
2008                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2009                                           "partial mask is not supported"
2010                                           " for protocol");
2011         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2012                                         acc_mask ? (const uint8_t *)acc_mask
2013                                                  : (const uint8_t *)&nic_mask,
2014                                         sizeof(struct rte_flow_item_ipv4),
2015                                         range_accepted, error);
2016         if (ret < 0)
2017                 return ret;
2018         return 0;
2019 }
2020
2021 /**
2022  * Validate IPV6 item.
2023  *
2024  * @param[in] item
2025  *   Item specification.
2026  * @param[in] item_flags
2027  *   Bit-fields that holds the items detected until now.
2028  * @param[in] last_item
2029  *   Previous validated item in the pattern items.
2030  * @param[in] ether_type
2031  *   Type in the ethernet layer header (including dot1q).
2032  * @param[in] acc_mask
2033  *   Acceptable mask, if NULL default internal default mask
2034  *   will be used to check whether item fields are supported.
2035  * @param[out] error
2036  *   Pointer to error structure.
2037  *
2038  * @return
2039  *   0 on success, a negative errno value otherwise and rte_errno is set.
2040  */
2041 int
2042 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2043                              uint64_t item_flags,
2044                              uint64_t last_item,
2045                              uint16_t ether_type,
2046                              const struct rte_flow_item_ipv6 *acc_mask,
2047                              struct rte_flow_error *error)
2048 {
2049         const struct rte_flow_item_ipv6 *mask = item->mask;
2050         const struct rte_flow_item_ipv6 *spec = item->spec;
2051         const struct rte_flow_item_ipv6 nic_mask = {
2052                 .hdr = {
2053                         .src_addr =
2054                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2055                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2056                         .dst_addr =
2057                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2058                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2059                         .vtc_flow = RTE_BE32(0xffffffff),
2060                         .proto = 0xff,
2061                 },
2062         };
2063         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2064         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2065                                       MLX5_FLOW_LAYER_OUTER_L3;
2066         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2067                                       MLX5_FLOW_LAYER_OUTER_L4;
2068         int ret;
2069         uint8_t next_proto = 0xFF;
2070         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2071                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2072                                   MLX5_FLOW_LAYER_INNER_VLAN);
2073
2074         if ((last_item & l2_vlan) && ether_type &&
2075             ether_type != RTE_ETHER_TYPE_IPV6)
2076                 return rte_flow_error_set(error, EINVAL,
2077                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2078                                           "IPv6 cannot follow L2/VLAN layer "
2079                                           "which ether type is not IPv6");
2080         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2081                 next_proto = spec->hdr.proto;
2082         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2083                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2084                         return rte_flow_error_set(error, EINVAL,
2085                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2086                                                   item,
2087                                                   "multiple tunnel "
2088                                                   "not supported");
2089         }
2090         if (next_proto == IPPROTO_HOPOPTS  ||
2091             next_proto == IPPROTO_ROUTING  ||
2092             next_proto == IPPROTO_FRAGMENT ||
2093             next_proto == IPPROTO_ESP      ||
2094             next_proto == IPPROTO_AH       ||
2095             next_proto == IPPROTO_DSTOPTS)
2096                 return rte_flow_error_set(error, EINVAL,
2097                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2098                                           "IPv6 proto (next header) should "
2099                                           "not be set as extension header");
2100         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2101                 return rte_flow_error_set(error, EINVAL,
2102                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2103                                           "wrong tunnel type - IPv4 specified "
2104                                           "but IPv6 item provided");
2105         if (item_flags & l3m)
2106                 return rte_flow_error_set(error, ENOTSUP,
2107                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2108                                           "multiple L3 layers not supported");
2109         else if (item_flags & l4m)
2110                 return rte_flow_error_set(error, EINVAL,
2111                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2112                                           "L3 cannot follow an L4 layer.");
2113         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2114                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2115                 return rte_flow_error_set(error, EINVAL,
2116                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2117                                           "L3 cannot follow an NVGRE layer.");
2118         if (!mask)
2119                 mask = &rte_flow_item_ipv6_mask;
2120         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2121                                         acc_mask ? (const uint8_t *)acc_mask
2122                                                  : (const uint8_t *)&nic_mask,
2123                                         sizeof(struct rte_flow_item_ipv6),
2124                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2125         if (ret < 0)
2126                 return ret;
2127         return 0;
2128 }
2129
2130 /**
2131  * Validate UDP item.
2132  *
2133  * @param[in] item
2134  *   Item specification.
2135  * @param[in] item_flags
2136  *   Bit-fields that holds the items detected until now.
2137  * @param[in] target_protocol
2138  *   The next protocol in the previous item.
2139  * @param[in] flow_mask
2140  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2141  * @param[out] error
2142  *   Pointer to error structure.
2143  *
2144  * @return
2145  *   0 on success, a negative errno value otherwise and rte_errno is set.
2146  */
2147 int
2148 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2149                             uint64_t item_flags,
2150                             uint8_t target_protocol,
2151                             struct rte_flow_error *error)
2152 {
2153         const struct rte_flow_item_udp *mask = item->mask;
2154         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2155         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2156                                       MLX5_FLOW_LAYER_OUTER_L3;
2157         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2158                                       MLX5_FLOW_LAYER_OUTER_L4;
2159         int ret;
2160
2161         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2162                 return rte_flow_error_set(error, EINVAL,
2163                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2164                                           "protocol filtering not compatible"
2165                                           " with UDP layer");
2166         if (!(item_flags & l3m))
2167                 return rte_flow_error_set(error, EINVAL,
2168                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2169                                           "L3 is mandatory to filter on L4");
2170         if (item_flags & l4m)
2171                 return rte_flow_error_set(error, EINVAL,
2172                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2173                                           "multiple L4 layers not supported");
2174         if (!mask)
2175                 mask = &rte_flow_item_udp_mask;
2176         ret = mlx5_flow_item_acceptable
2177                 (item, (const uint8_t *)mask,
2178                  (const uint8_t *)&rte_flow_item_udp_mask,
2179                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2180                  error);
2181         if (ret < 0)
2182                 return ret;
2183         return 0;
2184 }
2185
2186 /**
2187  * Validate TCP item.
2188  *
2189  * @param[in] item
2190  *   Item specification.
2191  * @param[in] item_flags
2192  *   Bit-fields that holds the items detected until now.
2193  * @param[in] target_protocol
2194  *   The next protocol in the previous item.
2195  * @param[out] error
2196  *   Pointer to error structure.
2197  *
2198  * @return
2199  *   0 on success, a negative errno value otherwise and rte_errno is set.
2200  */
2201 int
2202 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2203                             uint64_t item_flags,
2204                             uint8_t target_protocol,
2205                             const struct rte_flow_item_tcp *flow_mask,
2206                             struct rte_flow_error *error)
2207 {
2208         const struct rte_flow_item_tcp *mask = item->mask;
2209         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2210         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2211                                       MLX5_FLOW_LAYER_OUTER_L3;
2212         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2213                                       MLX5_FLOW_LAYER_OUTER_L4;
2214         int ret;
2215
2216         MLX5_ASSERT(flow_mask);
2217         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2218                 return rte_flow_error_set(error, EINVAL,
2219                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2220                                           "protocol filtering not compatible"
2221                                           " with TCP layer");
2222         if (!(item_flags & l3m))
2223                 return rte_flow_error_set(error, EINVAL,
2224                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2225                                           "L3 is mandatory to filter on L4");
2226         if (item_flags & l4m)
2227                 return rte_flow_error_set(error, EINVAL,
2228                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2229                                           "multiple L4 layers not supported");
2230         if (!mask)
2231                 mask = &rte_flow_item_tcp_mask;
2232         ret = mlx5_flow_item_acceptable
2233                 (item, (const uint8_t *)mask,
2234                  (const uint8_t *)flow_mask,
2235                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2236                  error);
2237         if (ret < 0)
2238                 return ret;
2239         return 0;
2240 }
2241
2242 /**
2243  * Validate VXLAN item.
2244  *
2245  * @param[in] item
2246  *   Item specification.
2247  * @param[in] item_flags
2248  *   Bit-fields that holds the items detected until now.
2249  * @param[in] target_protocol
2250  *   The next protocol in the previous item.
2251  * @param[out] error
2252  *   Pointer to error structure.
2253  *
2254  * @return
2255  *   0 on success, a negative errno value otherwise and rte_errno is set.
2256  */
2257 int
2258 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2259                               uint64_t item_flags,
2260                               struct rte_flow_error *error)
2261 {
2262         const struct rte_flow_item_vxlan *spec = item->spec;
2263         const struct rte_flow_item_vxlan *mask = item->mask;
2264         int ret;
2265         union vni {
2266                 uint32_t vlan_id;
2267                 uint8_t vni[4];
2268         } id = { .vlan_id = 0, };
2269
2270
2271         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2272                 return rte_flow_error_set(error, ENOTSUP,
2273                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2274                                           "multiple tunnel layers not"
2275                                           " supported");
2276         /*
2277          * Verify only UDPv4 is present as defined in
2278          * https://tools.ietf.org/html/rfc7348
2279          */
2280         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2281                 return rte_flow_error_set(error, EINVAL,
2282                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2283                                           "no outer UDP layer found");
2284         if (!mask)
2285                 mask = &rte_flow_item_vxlan_mask;
2286         ret = mlx5_flow_item_acceptable
2287                 (item, (const uint8_t *)mask,
2288                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2289                  sizeof(struct rte_flow_item_vxlan),
2290                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2291         if (ret < 0)
2292                 return ret;
2293         if (spec) {
2294                 memcpy(&id.vni[1], spec->vni, 3);
2295                 memcpy(&id.vni[1], mask->vni, 3);
2296         }
2297         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2298                 return rte_flow_error_set(error, ENOTSUP,
2299                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2300                                           "VXLAN tunnel must be fully defined");
2301         return 0;
2302 }
2303
2304 /**
2305  * Validate VXLAN_GPE item.
2306  *
2307  * @param[in] item
2308  *   Item specification.
2309  * @param[in] item_flags
2310  *   Bit-fields that holds the items detected until now.
2311  * @param[in] priv
2312  *   Pointer to the private data structure.
2313  * @param[in] target_protocol
2314  *   The next protocol in the previous item.
2315  * @param[out] error
2316  *   Pointer to error structure.
2317  *
2318  * @return
2319  *   0 on success, a negative errno value otherwise and rte_errno is set.
2320  */
2321 int
2322 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2323                                   uint64_t item_flags,
2324                                   struct rte_eth_dev *dev,
2325                                   struct rte_flow_error *error)
2326 {
2327         struct mlx5_priv *priv = dev->data->dev_private;
2328         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2329         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2330         int ret;
2331         union vni {
2332                 uint32_t vlan_id;
2333                 uint8_t vni[4];
2334         } id = { .vlan_id = 0, };
2335
2336         if (!priv->config.l3_vxlan_en)
2337                 return rte_flow_error_set(error, ENOTSUP,
2338                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2339                                           "L3 VXLAN is not enabled by device"
2340                                           " parameter and/or not configured in"
2341                                           " firmware");
2342         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2343                 return rte_flow_error_set(error, ENOTSUP,
2344                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2345                                           "multiple tunnel layers not"
2346                                           " supported");
2347         /*
2348          * Verify only UDPv4 is present as defined in
2349          * https://tools.ietf.org/html/rfc7348
2350          */
2351         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2352                 return rte_flow_error_set(error, EINVAL,
2353                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2354                                           "no outer UDP layer found");
2355         if (!mask)
2356                 mask = &rte_flow_item_vxlan_gpe_mask;
2357         ret = mlx5_flow_item_acceptable
2358                 (item, (const uint8_t *)mask,
2359                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2360                  sizeof(struct rte_flow_item_vxlan_gpe),
2361                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2362         if (ret < 0)
2363                 return ret;
2364         if (spec) {
2365                 if (spec->protocol)
2366                         return rte_flow_error_set(error, ENOTSUP,
2367                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2368                                                   item,
2369                                                   "VxLAN-GPE protocol"
2370                                                   " not supported");
2371                 memcpy(&id.vni[1], spec->vni, 3);
2372                 memcpy(&id.vni[1], mask->vni, 3);
2373         }
2374         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2375                 return rte_flow_error_set(error, ENOTSUP,
2376                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2377                                           "VXLAN-GPE tunnel must be fully"
2378                                           " defined");
2379         return 0;
2380 }
2381 /**
2382  * Validate GRE Key item.
2383  *
2384  * @param[in] item
2385  *   Item specification.
2386  * @param[in] item_flags
2387  *   Bit flags to mark detected items.
2388  * @param[in] gre_item
2389  *   Pointer to gre_item
2390  * @param[out] error
2391  *   Pointer to error structure.
2392  *
2393  * @return
2394  *   0 on success, a negative errno value otherwise and rte_errno is set.
2395  */
2396 int
2397 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2398                                 uint64_t item_flags,
2399                                 const struct rte_flow_item *gre_item,
2400                                 struct rte_flow_error *error)
2401 {
2402         const rte_be32_t *mask = item->mask;
2403         int ret = 0;
2404         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2405         const struct rte_flow_item_gre *gre_spec;
2406         const struct rte_flow_item_gre *gre_mask;
2407
2408         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2409                 return rte_flow_error_set(error, ENOTSUP,
2410                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2411                                           "Multiple GRE key not support");
2412         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2413                 return rte_flow_error_set(error, ENOTSUP,
2414                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2415                                           "No preceding GRE header");
2416         if (item_flags & MLX5_FLOW_LAYER_INNER)
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2419                                           "GRE key following a wrong item");
2420         gre_mask = gre_item->mask;
2421         if (!gre_mask)
2422                 gre_mask = &rte_flow_item_gre_mask;
2423         gre_spec = gre_item->spec;
2424         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2425                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2426                 return rte_flow_error_set(error, EINVAL,
2427                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2428                                           "Key bit must be on");
2429
2430         if (!mask)
2431                 mask = &gre_key_default_mask;
2432         ret = mlx5_flow_item_acceptable
2433                 (item, (const uint8_t *)mask,
2434                  (const uint8_t *)&gre_key_default_mask,
2435                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2436         return ret;
2437 }
2438
2439 /**
2440  * Validate GRE item.
2441  *
2442  * @param[in] item
2443  *   Item specification.
2444  * @param[in] item_flags
2445  *   Bit flags to mark detected items.
2446  * @param[in] target_protocol
2447  *   The next protocol in the previous item.
2448  * @param[out] error
2449  *   Pointer to error structure.
2450  *
2451  * @return
2452  *   0 on success, a negative errno value otherwise and rte_errno is set.
2453  */
2454 int
2455 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2456                             uint64_t item_flags,
2457                             uint8_t target_protocol,
2458                             struct rte_flow_error *error)
2459 {
2460         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2461         const struct rte_flow_item_gre *mask = item->mask;
2462         int ret;
2463         const struct rte_flow_item_gre nic_mask = {
2464                 .c_rsvd0_ver = RTE_BE16(0xB000),
2465                 .protocol = RTE_BE16(UINT16_MAX),
2466         };
2467
2468         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2469                 return rte_flow_error_set(error, EINVAL,
2470                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2471                                           "protocol filtering not compatible"
2472                                           " with this GRE layer");
2473         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2474                 return rte_flow_error_set(error, ENOTSUP,
2475                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2476                                           "multiple tunnel layers not"
2477                                           " supported");
2478         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2479                 return rte_flow_error_set(error, ENOTSUP,
2480                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2481                                           "L3 Layer is missing");
2482         if (!mask)
2483                 mask = &rte_flow_item_gre_mask;
2484         ret = mlx5_flow_item_acceptable
2485                 (item, (const uint8_t *)mask,
2486                  (const uint8_t *)&nic_mask,
2487                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2488                  error);
2489         if (ret < 0)
2490                 return ret;
2491 #ifndef HAVE_MLX5DV_DR
2492 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2493         if (spec && (spec->protocol & mask->protocol))
2494                 return rte_flow_error_set(error, ENOTSUP,
2495                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2496                                           "without MPLS support the"
2497                                           " specification cannot be used for"
2498                                           " filtering");
2499 #endif
2500 #endif
2501         return 0;
2502 }
2503
2504 /**
2505  * Validate Geneve item.
2506  *
2507  * @param[in] item
2508  *   Item specification.
2509  * @param[in] itemFlags
2510  *   Bit-fields that holds the items detected until now.
2511  * @param[in] enPriv
2512  *   Pointer to the private data structure.
2513  * @param[out] error
2514  *   Pointer to error structure.
2515  *
2516  * @return
2517  *   0 on success, a negative errno value otherwise and rte_errno is set.
2518  */
2519
2520 int
2521 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2522                                uint64_t item_flags,
2523                                struct rte_eth_dev *dev,
2524                                struct rte_flow_error *error)
2525 {
2526         struct mlx5_priv *priv = dev->data->dev_private;
2527         const struct rte_flow_item_geneve *spec = item->spec;
2528         const struct rte_flow_item_geneve *mask = item->mask;
2529         int ret;
2530         uint16_t gbhdr;
2531         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2532                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2533         const struct rte_flow_item_geneve nic_mask = {
2534                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2535                 .vni = "\xff\xff\xff",
2536                 .protocol = RTE_BE16(UINT16_MAX),
2537         };
2538
2539         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2540                 return rte_flow_error_set(error, ENOTSUP,
2541                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2542                                           "L3 Geneve is not enabled by device"
2543                                           " parameter and/or not configured in"
2544                                           " firmware");
2545         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2546                 return rte_flow_error_set(error, ENOTSUP,
2547                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2548                                           "multiple tunnel layers not"
2549                                           " supported");
2550         /*
2551          * Verify only UDPv4 is present as defined in
2552          * https://tools.ietf.org/html/rfc7348
2553          */
2554         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2555                 return rte_flow_error_set(error, EINVAL,
2556                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2557                                           "no outer UDP layer found");
2558         if (!mask)
2559                 mask = &rte_flow_item_geneve_mask;
2560         ret = mlx5_flow_item_acceptable
2561                                   (item, (const uint8_t *)mask,
2562                                    (const uint8_t *)&nic_mask,
2563                                    sizeof(struct rte_flow_item_geneve),
2564                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2565         if (ret)
2566                 return ret;
2567         if (spec) {
2568                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2569                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2570                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2571                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2572                         return rte_flow_error_set(error, ENOTSUP,
2573                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2574                                                   item,
2575                                                   "Geneve protocol unsupported"
2576                                                   " fields are being used");
2577                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2578                         return rte_flow_error_set
2579                                         (error, ENOTSUP,
2580                                          RTE_FLOW_ERROR_TYPE_ITEM,
2581                                          item,
2582                                          "Unsupported Geneve options length");
2583         }
2584         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2585                 return rte_flow_error_set
2586                                     (error, ENOTSUP,
2587                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2588                                      "Geneve tunnel must be fully defined");
2589         return 0;
2590 }
2591
2592 /**
2593  * Validate MPLS item.
2594  *
2595  * @param[in] dev
2596  *   Pointer to the rte_eth_dev structure.
2597  * @param[in] item
2598  *   Item specification.
2599  * @param[in] item_flags
2600  *   Bit-fields that holds the items detected until now.
2601  * @param[in] prev_layer
2602  *   The protocol layer indicated in previous item.
2603  * @param[out] error
2604  *   Pointer to error structure.
2605  *
2606  * @return
2607  *   0 on success, a negative errno value otherwise and rte_errno is set.
2608  */
2609 int
2610 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2611                              const struct rte_flow_item *item __rte_unused,
2612                              uint64_t item_flags __rte_unused,
2613                              uint64_t prev_layer __rte_unused,
2614                              struct rte_flow_error *error)
2615 {
2616 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2617         const struct rte_flow_item_mpls *mask = item->mask;
2618         struct mlx5_priv *priv = dev->data->dev_private;
2619         int ret;
2620
2621         if (!priv->config.mpls_en)
2622                 return rte_flow_error_set(error, ENOTSUP,
2623                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2624                                           "MPLS not supported or"
2625                                           " disabled in firmware"
2626                                           " configuration.");
2627         /* MPLS over IP, UDP, GRE is allowed */
2628         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2629                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2630                             MLX5_FLOW_LAYER_GRE |
2631                             MLX5_FLOW_LAYER_GRE_KEY)))
2632                 return rte_flow_error_set(error, EINVAL,
2633                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2634                                           "protocol filtering not compatible"
2635                                           " with MPLS layer");
2636         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2637         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2638             !(item_flags & MLX5_FLOW_LAYER_GRE))
2639                 return rte_flow_error_set(error, ENOTSUP,
2640                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2641                                           "multiple tunnel layers not"
2642                                           " supported");
2643         if (!mask)
2644                 mask = &rte_flow_item_mpls_mask;
2645         ret = mlx5_flow_item_acceptable
2646                 (item, (const uint8_t *)mask,
2647                  (const uint8_t *)&rte_flow_item_mpls_mask,
2648                  sizeof(struct rte_flow_item_mpls),
2649                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2650         if (ret < 0)
2651                 return ret;
2652         return 0;
2653 #else
2654         return rte_flow_error_set(error, ENOTSUP,
2655                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2656                                   "MPLS is not supported by Verbs, please"
2657                                   " update.");
2658 #endif
2659 }
2660
2661 /**
2662  * Validate NVGRE item.
2663  *
2664  * @param[in] item
2665  *   Item specification.
2666  * @param[in] item_flags
2667  *   Bit flags to mark detected items.
2668  * @param[in] target_protocol
2669  *   The next protocol in the previous item.
2670  * @param[out] error
2671  *   Pointer to error structure.
2672  *
2673  * @return
2674  *   0 on success, a negative errno value otherwise and rte_errno is set.
2675  */
2676 int
2677 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2678                               uint64_t item_flags,
2679                               uint8_t target_protocol,
2680                               struct rte_flow_error *error)
2681 {
2682         const struct rte_flow_item_nvgre *mask = item->mask;
2683         int ret;
2684
2685         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2686                 return rte_flow_error_set(error, EINVAL,
2687                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2688                                           "protocol filtering not compatible"
2689                                           " with this GRE layer");
2690         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2691                 return rte_flow_error_set(error, ENOTSUP,
2692                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2693                                           "multiple tunnel layers not"
2694                                           " supported");
2695         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2696                 return rte_flow_error_set(error, ENOTSUP,
2697                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2698                                           "L3 Layer is missing");
2699         if (!mask)
2700                 mask = &rte_flow_item_nvgre_mask;
2701         ret = mlx5_flow_item_acceptable
2702                 (item, (const uint8_t *)mask,
2703                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2704                  sizeof(struct rte_flow_item_nvgre),
2705                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2706         if (ret < 0)
2707                 return ret;
2708         return 0;
2709 }
2710
2711 /**
2712  * Validate eCPRI item.
2713  *
2714  * @param[in] item
2715  *   Item specification.
2716  * @param[in] item_flags
2717  *   Bit-fields that holds the items detected until now.
2718  * @param[in] last_item
2719  *   Previous validated item in the pattern items.
2720  * @param[in] ether_type
2721  *   Type in the ethernet layer header (including dot1q).
2722  * @param[in] acc_mask
2723  *   Acceptable mask, if NULL default internal default mask
2724  *   will be used to check whether item fields are supported.
2725  * @param[out] error
2726  *   Pointer to error structure.
2727  *
2728  * @return
2729  *   0 on success, a negative errno value otherwise and rte_errno is set.
2730  */
2731 int
2732 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2733                               uint64_t item_flags,
2734                               uint64_t last_item,
2735                               uint16_t ether_type,
2736                               const struct rte_flow_item_ecpri *acc_mask,
2737                               struct rte_flow_error *error)
2738 {
2739         const struct rte_flow_item_ecpri *mask = item->mask;
2740         const struct rte_flow_item_ecpri nic_mask = {
2741                 .hdr = {
2742                         .common = {
2743                                 .u32 =
2744                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2745                                         .type = 0xFF,
2746                                         }).u32),
2747                         },
2748                         .dummy[0] = 0xFFFFFFFF,
2749                 },
2750         };
2751         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2752                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2753         struct rte_flow_item_ecpri mask_lo;
2754
2755         if (!(last_item & outer_l2_vlan) &&
2756             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
2757                 return rte_flow_error_set(error, EINVAL,
2758                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2759                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
2760         if ((last_item & outer_l2_vlan) && ether_type &&
2761             ether_type != RTE_ETHER_TYPE_ECPRI)
2762                 return rte_flow_error_set(error, EINVAL,
2763                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2764                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
2765         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2766                 return rte_flow_error_set(error, EINVAL,
2767                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2768                                           "eCPRI with tunnel is not supported right now");
2769         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2770                 return rte_flow_error_set(error, ENOTSUP,
2771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                                           "multiple L3 layers not supported");
2773         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2774                 return rte_flow_error_set(error, EINVAL,
2775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2776                                           "eCPRI cannot coexist with a TCP layer");
2777         /* In specification, eCPRI could be over UDP layer. */
2778         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2779                 return rte_flow_error_set(error, EINVAL,
2780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2781                                           "eCPRI over UDP layer is not yet supported right now");
2782         /* Mask for type field in common header could be zero. */
2783         if (!mask)
2784                 mask = &rte_flow_item_ecpri_mask;
2785         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2786         /* Input mask is in big-endian format. */
2787         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2788                 return rte_flow_error_set(error, EINVAL,
2789                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2790                                           "partial mask is not supported for protocol");
2791         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2792                 return rte_flow_error_set(error, EINVAL,
2793                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2794                                           "message header mask must be after a type mask");
2795         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2796                                          acc_mask ? (const uint8_t *)acc_mask
2797                                                   : (const uint8_t *)&nic_mask,
2798                                          sizeof(struct rte_flow_item_ecpri),
2799                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2800 }
2801
2802 /**
2803  * Release resource related QUEUE/RSS action split.
2804  *
2805  * @param dev
2806  *   Pointer to Ethernet device.
2807  * @param flow
2808  *   Flow to release id's from.
2809  */
2810 static void
2811 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2812                              struct rte_flow *flow)
2813 {
2814         struct mlx5_priv *priv = dev->data->dev_private;
2815         uint32_t handle_idx;
2816         struct mlx5_flow_handle *dev_handle;
2817
2818         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2819                        handle_idx, dev_handle, next)
2820                 if (dev_handle->split_flow_id)
2821                         mlx5_ipool_free(priv->sh->ipool
2822                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
2823                                         dev_handle->split_flow_id);
2824 }
2825
2826 static int
2827 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2828                    const struct rte_flow_attr *attr __rte_unused,
2829                    const struct rte_flow_item items[] __rte_unused,
2830                    const struct rte_flow_action actions[] __rte_unused,
2831                    bool external __rte_unused,
2832                    int hairpin __rte_unused,
2833                    struct rte_flow_error *error)
2834 {
2835         return rte_flow_error_set(error, ENOTSUP,
2836                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2837 }
2838
2839 static struct mlx5_flow *
2840 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2841                   const struct rte_flow_attr *attr __rte_unused,
2842                   const struct rte_flow_item items[] __rte_unused,
2843                   const struct rte_flow_action actions[] __rte_unused,
2844                   struct rte_flow_error *error)
2845 {
2846         rte_flow_error_set(error, ENOTSUP,
2847                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2848         return NULL;
2849 }
2850
2851 static int
2852 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2853                     struct mlx5_flow *dev_flow __rte_unused,
2854                     const struct rte_flow_attr *attr __rte_unused,
2855                     const struct rte_flow_item items[] __rte_unused,
2856                     const struct rte_flow_action actions[] __rte_unused,
2857                     struct rte_flow_error *error)
2858 {
2859         return rte_flow_error_set(error, ENOTSUP,
2860                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2861 }
2862
2863 static int
2864 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2865                 struct rte_flow *flow __rte_unused,
2866                 struct rte_flow_error *error)
2867 {
2868         return rte_flow_error_set(error, ENOTSUP,
2869                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2870 }
2871
2872 static void
2873 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2874                  struct rte_flow *flow __rte_unused)
2875 {
2876 }
2877
2878 static void
2879 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2880                   struct rte_flow *flow __rte_unused)
2881 {
2882 }
2883
2884 static int
2885 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2886                 struct rte_flow *flow __rte_unused,
2887                 const struct rte_flow_action *actions __rte_unused,
2888                 void *data __rte_unused,
2889                 struct rte_flow_error *error)
2890 {
2891         return rte_flow_error_set(error, ENOTSUP,
2892                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2893 }
2894
2895 static int
2896 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
2897                       uint32_t domains __rte_unused,
2898                       uint32_t flags __rte_unused)
2899 {
2900         return 0;
2901 }
2902
2903 /* Void driver to protect from null pointer reference. */
2904 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2905         .validate = flow_null_validate,
2906         .prepare = flow_null_prepare,
2907         .translate = flow_null_translate,
2908         .apply = flow_null_apply,
2909         .remove = flow_null_remove,
2910         .destroy = flow_null_destroy,
2911         .query = flow_null_query,
2912         .sync_domain = flow_null_sync_domain,
2913 };
2914
2915 /**
2916  * Select flow driver type according to flow attributes and device
2917  * configuration.
2918  *
2919  * @param[in] dev
2920  *   Pointer to the dev structure.
2921  * @param[in] attr
2922  *   Pointer to the flow attributes.
2923  *
2924  * @return
2925  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2926  */
2927 static enum mlx5_flow_drv_type
2928 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2929 {
2930         struct mlx5_priv *priv = dev->data->dev_private;
2931         /* The OS can determine first a specific flow type (DV, VERBS) */
2932         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2933
2934         if (type != MLX5_FLOW_TYPE_MAX)
2935                 return type;
2936         /* If no OS specific type - continue with DV/VERBS selection */
2937         if (attr->transfer && priv->config.dv_esw_en)
2938                 type = MLX5_FLOW_TYPE_DV;
2939         if (!attr->transfer)
2940                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2941                                                  MLX5_FLOW_TYPE_VERBS;
2942         return type;
2943 }
2944
2945 #define flow_get_drv_ops(type) flow_drv_ops[type]
2946
2947 /**
2948  * Flow driver validation API. This abstracts calling driver specific functions.
2949  * The type of flow driver is determined according to flow attributes.
2950  *
2951  * @param[in] dev
2952  *   Pointer to the dev structure.
2953  * @param[in] attr
2954  *   Pointer to the flow attributes.
2955  * @param[in] items
2956  *   Pointer to the list of items.
2957  * @param[in] actions
2958  *   Pointer to the list of actions.
2959  * @param[in] external
2960  *   This flow rule is created by request external to PMD.
2961  * @param[in] hairpin
2962  *   Number of hairpin TX actions, 0 means classic flow.
2963  * @param[out] error
2964  *   Pointer to the error structure.
2965  *
2966  * @return
2967  *   0 on success, a negative errno value otherwise and rte_errno is set.
2968  */
2969 static inline int
2970 flow_drv_validate(struct rte_eth_dev *dev,
2971                   const struct rte_flow_attr *attr,
2972                   const struct rte_flow_item items[],
2973                   const struct rte_flow_action actions[],
2974                   bool external, int hairpin, struct rte_flow_error *error)
2975 {
2976         const struct mlx5_flow_driver_ops *fops;
2977         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2978
2979         fops = flow_get_drv_ops(type);
2980         return fops->validate(dev, attr, items, actions, external,
2981                               hairpin, error);
2982 }
2983
2984 /**
2985  * Flow driver preparation API. This abstracts calling driver specific
2986  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2987  * calculates the size of memory required for device flow, allocates the memory,
2988  * initializes the device flow and returns the pointer.
2989  *
2990  * @note
2991  *   This function initializes device flow structure such as dv or verbs in
2992  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
2993  *   rest. For example, adding returning device flow to flow->dev_flow list and
2994  *   setting backward reference to the flow should be done out of this function.
2995  *   layers field is not filled either.
2996  *
2997  * @param[in] dev
2998  *   Pointer to the dev structure.
2999  * @param[in] attr
3000  *   Pointer to the flow attributes.
3001  * @param[in] items
3002  *   Pointer to the list of items.
3003  * @param[in] actions
3004  *   Pointer to the list of actions.
3005  * @param[in] flow_idx
3006  *   This memory pool index to the flow.
3007  * @param[out] error
3008  *   Pointer to the error structure.
3009  *
3010  * @return
3011  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3012  */
3013 static inline struct mlx5_flow *
3014 flow_drv_prepare(struct rte_eth_dev *dev,
3015                  const struct rte_flow *flow,
3016                  const struct rte_flow_attr *attr,
3017                  const struct rte_flow_item items[],
3018                  const struct rte_flow_action actions[],
3019                  uint32_t flow_idx,
3020                  struct rte_flow_error *error)
3021 {
3022         const struct mlx5_flow_driver_ops *fops;
3023         enum mlx5_flow_drv_type type = flow->drv_type;
3024         struct mlx5_flow *mlx5_flow = NULL;
3025
3026         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3027         fops = flow_get_drv_ops(type);
3028         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3029         if (mlx5_flow)
3030                 mlx5_flow->flow_idx = flow_idx;
3031         return mlx5_flow;
3032 }
3033
3034 /**
3035  * Flow driver translation API. This abstracts calling driver specific
3036  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3037  * translates a generic flow into a driver flow. flow_drv_prepare() must
3038  * precede.
3039  *
3040  * @note
3041  *   dev_flow->layers could be filled as a result of parsing during translation
3042  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3043  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3044  *   flow->actions could be overwritten even though all the expanded dev_flows
3045  *   have the same actions.
3046  *
3047  * @param[in] dev
3048  *   Pointer to the rte dev structure.
3049  * @param[in, out] dev_flow
3050  *   Pointer to the mlx5 flow.
3051  * @param[in] attr
3052  *   Pointer to the flow attributes.
3053  * @param[in] items
3054  *   Pointer to the list of items.
3055  * @param[in] actions
3056  *   Pointer to the list of actions.
3057  * @param[out] error
3058  *   Pointer to the error structure.
3059  *
3060  * @return
3061  *   0 on success, a negative errno value otherwise and rte_errno is set.
3062  */
3063 static inline int
3064 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3065                    const struct rte_flow_attr *attr,
3066                    const struct rte_flow_item items[],
3067                    const struct rte_flow_action actions[],
3068                    struct rte_flow_error *error)
3069 {
3070         const struct mlx5_flow_driver_ops *fops;
3071         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3072
3073         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3074         fops = flow_get_drv_ops(type);
3075         return fops->translate(dev, dev_flow, attr, items, actions, error);
3076 }
3077
3078 /**
3079  * Flow driver apply API. This abstracts calling driver specific functions.
3080  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3081  * translated driver flows on to device. flow_drv_translate() must precede.
3082  *
3083  * @param[in] dev
3084  *   Pointer to Ethernet device structure.
3085  * @param[in, out] flow
3086  *   Pointer to flow structure.
3087  * @param[out] error
3088  *   Pointer to error structure.
3089  *
3090  * @return
3091  *   0 on success, a negative errno value otherwise and rte_errno is set.
3092  */
3093 static inline int
3094 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3095                struct rte_flow_error *error)
3096 {
3097         const struct mlx5_flow_driver_ops *fops;
3098         enum mlx5_flow_drv_type type = flow->drv_type;
3099
3100         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3101         fops = flow_get_drv_ops(type);
3102         return fops->apply(dev, flow, error);
3103 }
3104
3105 /**
3106  * Flow driver destroy API. This abstracts calling driver specific functions.
3107  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3108  * on device and releases resources of the flow.
3109  *
3110  * @param[in] dev
3111  *   Pointer to Ethernet device.
3112  * @param[in, out] flow
3113  *   Pointer to flow structure.
3114  */
3115 static inline void
3116 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3117 {
3118         const struct mlx5_flow_driver_ops *fops;
3119         enum mlx5_flow_drv_type type = flow->drv_type;
3120
3121         flow_mreg_split_qrss_release(dev, flow);
3122         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3123         fops = flow_get_drv_ops(type);
3124         fops->destroy(dev, flow);
3125 }
3126
3127 /**
3128  * Get RSS action from the action list.
3129  *
3130  * @param[in] actions
3131  *   Pointer to the list of actions.
3132  *
3133  * @return
3134  *   Pointer to the RSS action if exist, else return NULL.
3135  */
3136 static const struct rte_flow_action_rss*
3137 flow_get_rss_action(const struct rte_flow_action actions[])
3138 {
3139         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3140                 switch (actions->type) {
3141                 case RTE_FLOW_ACTION_TYPE_RSS:
3142                         return (const struct rte_flow_action_rss *)
3143                                actions->conf;
3144                 default:
3145                         break;
3146                 }
3147         }
3148         return NULL;
3149 }
3150
3151 /**
3152  * Get ASO age action by index.
3153  *
3154  * @param[in] dev
3155  *   Pointer to the Ethernet device structure.
3156  * @param[in] age_idx
3157  *   Index to the ASO age action.
3158  *
3159  * @return
3160  *   The specified ASO age action.
3161  */
3162 struct mlx5_aso_age_action*
3163 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3164 {
3165         uint16_t pool_idx = age_idx & UINT16_MAX;
3166         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3167         struct mlx5_priv *priv = dev->data->dev_private;
3168         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3169         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3170
3171         return &pool->actions[offset - 1];
3172 }
3173
3174 /* maps shared action to translated non shared in some actions array */
3175 struct mlx5_translated_shared_action {
3176         struct rte_flow_shared_action *action; /**< Shared action */
3177         int index; /**< Index in related array of rte_flow_action */
3178 };
3179
3180 /**
3181  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3182  * non shared action if translation possible.
3183  * This functionality used to run same execution path for both shared & non
3184  * shared actions on flow create. All necessary preparations for shared
3185  * action handling should be preformed on *shared* actions list returned
3186  * from this call.
3187  *
3188  * @param[in] dev
3189  *   Pointer to Ethernet device.
3190  * @param[in] actions
3191  *   List of actions to translate.
3192  * @param[out] shared
3193  *   List to store translated shared actions.
3194  * @param[in, out] shared_n
3195  *   Size of *shared* array. On return should be updated with number of shared
3196  *   actions retrieved from the *actions* list.
3197  * @param[out] translated_actions
3198  *   List of actions where all shared actions were translated to non shared
3199  *   if possible. NULL if no translation took place.
3200  * @param[out] error
3201  *   Pointer to the error structure.
3202  *
3203  * @return
3204  *   0 on success, a negative errno value otherwise and rte_errno is set.
3205  */
3206 static int
3207 flow_shared_actions_translate(struct rte_eth_dev *dev,
3208                               const struct rte_flow_action actions[],
3209                               struct mlx5_translated_shared_action *shared,
3210                               int *shared_n,
3211                               struct rte_flow_action **translated_actions,
3212                               struct rte_flow_error *error)
3213 {
3214         struct mlx5_priv *priv = dev->data->dev_private;
3215         struct rte_flow_action *translated = NULL;
3216         size_t actions_size;
3217         int n;
3218         int copied_n = 0;
3219         struct mlx5_translated_shared_action *shared_end = NULL;
3220
3221         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3222                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3223                         continue;
3224                 if (copied_n == *shared_n) {
3225                         return rte_flow_error_set
3226                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3227                                  NULL, "too many shared actions");
3228                 }
3229                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3230                            sizeof(actions[n].conf));
3231                 shared[copied_n].index = n;
3232                 copied_n++;
3233         }
3234         n++;
3235         *shared_n = copied_n;
3236         if (!copied_n)
3237                 return 0;
3238         actions_size = sizeof(struct rte_flow_action) * n;
3239         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3240         if (!translated) {
3241                 rte_errno = ENOMEM;
3242                 return -ENOMEM;
3243         }
3244         memcpy(translated, actions, actions_size);
3245         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3246                 struct mlx5_shared_action_rss *shared_rss;
3247                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3248                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3249                 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET)
3250                                                                            - 1);
3251
3252                 switch (type) {
3253                 case MLX5_SHARED_ACTION_TYPE_RSS:
3254                         shared_rss = mlx5_ipool_get
3255                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3256                         translated[shared->index].type =
3257                                 RTE_FLOW_ACTION_TYPE_RSS;
3258                         translated[shared->index].conf =
3259                                 &shared_rss->origin;
3260                         break;
3261                 case MLX5_SHARED_ACTION_TYPE_AGE:
3262                         if (priv->sh->flow_hit_aso_en) {
3263                                 translated[shared->index].type =
3264                                         (enum rte_flow_action_type)
3265                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3266                                 translated[shared->index].conf =
3267                                                          (void *)(uintptr_t)idx;
3268                                 break;
3269                         }
3270                         /* Fall-through */
3271                 default:
3272                         mlx5_free(translated);
3273                         return rte_flow_error_set
3274                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3275                                  NULL, "invalid shared action type");
3276                 }
3277         }
3278         *translated_actions = translated;
3279         return 0;
3280 }
3281
3282 /**
3283  * Get Shared RSS action from the action list.
3284  *
3285  * @param[in] dev
3286  *   Pointer to Ethernet device.
3287  * @param[in] shared
3288  *   Pointer to the list of actions.
3289  * @param[in] shared_n
3290  *   Actions list length.
3291  *
3292  * @return
3293  *   The MLX5 RSS action ID if exists, otherwise return 0.
3294  */
3295 static uint32_t
3296 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3297                            struct mlx5_translated_shared_action *shared,
3298                            int shared_n)
3299 {
3300         struct mlx5_translated_shared_action *shared_end;
3301         struct mlx5_priv *priv = dev->data->dev_private;
3302         struct mlx5_shared_action_rss *shared_rss;
3303
3304
3305         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3306                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3307                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3308                 uint32_t idx = act_idx &
3309                                    ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1);
3310                 switch (type) {
3311                 case MLX5_SHARED_ACTION_TYPE_RSS:
3312                         shared_rss = mlx5_ipool_get
3313                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3314                                                                            idx);
3315                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3316                                            __ATOMIC_RELAXED);
3317                         return idx;
3318                 default:
3319                         break;
3320                 }
3321         }
3322         return 0;
3323 }
3324
3325 static unsigned int
3326 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3327 {
3328         const struct rte_flow_item *item;
3329         unsigned int has_vlan = 0;
3330
3331         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3332                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3333                         has_vlan = 1;
3334                         break;
3335                 }
3336         }
3337         if (has_vlan)
3338                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3339                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3340         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3341                                MLX5_EXPANSION_ROOT_OUTER;
3342 }
3343
3344 /**
3345  *  Get layer flags from the prefix flow.
3346  *
3347  *  Some flows may be split to several subflows, the prefix subflow gets the
3348  *  match items and the suffix sub flow gets the actions.
3349  *  Some actions need the user defined match item flags to get the detail for
3350  *  the action.
3351  *  This function helps the suffix flow to get the item layer flags from prefix
3352  *  subflow.
3353  *
3354  * @param[in] dev_flow
3355  *   Pointer the created preifx subflow.
3356  *
3357  * @return
3358  *   The layers get from prefix subflow.
3359  */
3360 static inline uint64_t
3361 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3362 {
3363         uint64_t layers = 0;
3364
3365         /*
3366          * Layers bits could be localization, but usually the compiler will
3367          * help to do the optimization work for source code.
3368          * If no decap actions, use the layers directly.
3369          */
3370         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3371                 return dev_flow->handle->layers;
3372         /* Convert L3 layers with decap action. */
3373         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3374                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3375         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3376                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3377         /* Convert L4 layers with decap action.  */
3378         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3379                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3380         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3381                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3382         return layers;
3383 }
3384
3385 /**
3386  * Get metadata split action information.
3387  *
3388  * @param[in] actions
3389  *   Pointer to the list of actions.
3390  * @param[out] qrss
3391  *   Pointer to the return pointer.
3392  * @param[out] qrss_type
3393  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3394  *   if no QUEUE/RSS is found.
3395  * @param[out] encap_idx
3396  *   Pointer to the index of the encap action if exists, otherwise the last
3397  *   action index.
3398  *
3399  * @return
3400  *   Total number of actions.
3401  */
3402 static int
3403 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3404                                        const struct rte_flow_action **qrss,
3405                                        int *encap_idx)
3406 {
3407         const struct rte_flow_action_raw_encap *raw_encap;
3408         int actions_n = 0;
3409         int raw_decap_idx = -1;
3410
3411         *encap_idx = -1;
3412         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3413                 switch (actions->type) {
3414                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3415                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3416                         *encap_idx = actions_n;
3417                         break;
3418                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3419                         raw_decap_idx = actions_n;
3420                         break;
3421                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3422                         raw_encap = actions->conf;
3423                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3424                                 *encap_idx = raw_decap_idx != -1 ?
3425                                                       raw_decap_idx : actions_n;
3426                         break;
3427                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3428                 case RTE_FLOW_ACTION_TYPE_RSS:
3429                         *qrss = actions;
3430                         break;
3431                 default:
3432                         break;
3433                 }
3434                 actions_n++;
3435         }
3436         if (*encap_idx == -1)
3437                 *encap_idx = actions_n;
3438         /* Count RTE_FLOW_ACTION_TYPE_END. */
3439         return actions_n + 1;
3440 }
3441
3442 /**
3443  * Check meter action from the action list.
3444  *
3445  * @param[in] actions
3446  *   Pointer to the list of actions.
3447  * @param[out] mtr
3448  *   Pointer to the meter exist flag.
3449  *
3450  * @return
3451  *   Total number of actions.
3452  */
3453 static int
3454 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3455 {
3456         int actions_n = 0;
3457
3458         MLX5_ASSERT(mtr);
3459         *mtr = 0;
3460         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3461                 switch (actions->type) {
3462                 case RTE_FLOW_ACTION_TYPE_METER:
3463                         *mtr = 1;
3464                         break;
3465                 default:
3466                         break;
3467                 }
3468                 actions_n++;
3469         }
3470         /* Count RTE_FLOW_ACTION_TYPE_END. */
3471         return actions_n + 1;
3472 }
3473
3474 /**
3475  * Check if the flow should be split due to hairpin.
3476  * The reason for the split is that in current HW we can't
3477  * support encap and push-vlan on Rx, so if a flow contains
3478  * these actions we move it to Tx.
3479  *
3480  * @param dev
3481  *   Pointer to Ethernet device.
3482  * @param[in] attr
3483  *   Flow rule attributes.
3484  * @param[in] actions
3485  *   Associated actions (list terminated by the END action).
3486  *
3487  * @return
3488  *   > 0 the number of actions and the flow should be split,
3489  *   0 when no split required.
3490  */
3491 static int
3492 flow_check_hairpin_split(struct rte_eth_dev *dev,
3493                          const struct rte_flow_attr *attr,
3494                          const struct rte_flow_action actions[])
3495 {
3496         int queue_action = 0;
3497         int action_n = 0;
3498         int split = 0;
3499         const struct rte_flow_action_queue *queue;
3500         const struct rte_flow_action_rss *rss;
3501         const struct rte_flow_action_raw_encap *raw_encap;
3502         const struct rte_eth_hairpin_conf *conf;
3503
3504         if (!attr->ingress)
3505                 return 0;
3506         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3507                 switch (actions->type) {
3508                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3509                         queue = actions->conf;
3510                         if (queue == NULL)
3511                                 return 0;
3512                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3513                         if (conf != NULL && !!conf->tx_explicit)
3514                                 return 0;
3515                         queue_action = 1;
3516                         action_n++;
3517                         break;
3518                 case RTE_FLOW_ACTION_TYPE_RSS:
3519                         rss = actions->conf;
3520                         if (rss == NULL || rss->queue_num == 0)
3521                                 return 0;
3522                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3523                         if (conf != NULL && !!conf->tx_explicit)
3524                                 return 0;
3525                         queue_action = 1;
3526                         action_n++;
3527                         break;
3528                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3529                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3530                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3531                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3532                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3533                         split++;
3534                         action_n++;
3535                         break;
3536                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3537                         raw_encap = actions->conf;
3538                         if (raw_encap->size >
3539                             (sizeof(struct rte_flow_item_eth) +
3540                              sizeof(struct rte_flow_item_ipv4)))
3541                                 split++;
3542                         action_n++;
3543                         break;
3544                 default:
3545                         action_n++;
3546                         break;
3547                 }
3548         }
3549         if (split && queue_action)
3550                 return action_n;
3551         return 0;
3552 }
3553
3554 /* Declare flow create/destroy prototype in advance. */
3555 static uint32_t
3556 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3557                  const struct rte_flow_attr *attr,
3558                  const struct rte_flow_item items[],
3559                  const struct rte_flow_action actions[],
3560                  bool external, struct rte_flow_error *error);
3561
3562 static void
3563 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3564                   uint32_t flow_idx);
3565
3566 struct mlx5_hlist_entry *
3567 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3568                        void *cb_ctx)
3569 {
3570         struct rte_eth_dev *dev = list->ctx;
3571         struct mlx5_priv *priv = dev->data->dev_private;
3572         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3573         struct mlx5_flow_mreg_copy_resource *mcp_res;
3574         struct rte_flow_error *error = ctx->error;
3575         uint32_t idx = 0;
3576         int ret;
3577         uint32_t mark_id = key;
3578         struct rte_flow_attr attr = {
3579                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3580                 .ingress = 1,
3581         };
3582         struct mlx5_rte_flow_item_tag tag_spec = {
3583                 .data = mark_id,
3584         };
3585         struct rte_flow_item items[] = {
3586                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3587         };
3588         struct rte_flow_action_mark ftag = {
3589                 .id = mark_id,
3590         };
3591         struct mlx5_flow_action_copy_mreg cp_mreg = {
3592                 .dst = REG_B,
3593                 .src = REG_NON,
3594         };
3595         struct rte_flow_action_jump jump = {
3596                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3597         };
3598         struct rte_flow_action actions[] = {
3599                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3600         };
3601
3602         /* Fill the register fileds in the flow. */
3603         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3604         if (ret < 0)
3605                 return NULL;
3606         tag_spec.id = ret;
3607         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3608         if (ret < 0)
3609                 return NULL;
3610         cp_mreg.src = ret;
3611         /* Provide the full width of FLAG specific value. */
3612         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3613                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3614         /* Build a new flow. */
3615         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3616                 items[0] = (struct rte_flow_item){
3617                         .type = (enum rte_flow_item_type)
3618                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3619                         .spec = &tag_spec,
3620                 };
3621                 items[1] = (struct rte_flow_item){
3622                         .type = RTE_FLOW_ITEM_TYPE_END,
3623                 };
3624                 actions[0] = (struct rte_flow_action){
3625                         .type = (enum rte_flow_action_type)
3626                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3627                         .conf = &ftag,
3628                 };
3629                 actions[1] = (struct rte_flow_action){
3630                         .type = (enum rte_flow_action_type)
3631                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3632                         .conf = &cp_mreg,
3633                 };
3634                 actions[2] = (struct rte_flow_action){
3635                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3636                         .conf = &jump,
3637                 };
3638                 actions[3] = (struct rte_flow_action){
3639                         .type = RTE_FLOW_ACTION_TYPE_END,
3640                 };
3641         } else {
3642                 /* Default rule, wildcard match. */
3643                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3644                 items[0] = (struct rte_flow_item){
3645                         .type = RTE_FLOW_ITEM_TYPE_END,
3646                 };
3647                 actions[0] = (struct rte_flow_action){
3648                         .type = (enum rte_flow_action_type)
3649                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3650                         .conf = &cp_mreg,
3651                 };
3652                 actions[1] = (struct rte_flow_action){
3653                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3654                         .conf = &jump,
3655                 };
3656                 actions[2] = (struct rte_flow_action){
3657                         .type = RTE_FLOW_ACTION_TYPE_END,
3658                 };
3659         }
3660         /* Build a new entry. */
3661         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3662         if (!mcp_res) {
3663                 rte_errno = ENOMEM;
3664                 return NULL;
3665         }
3666         mcp_res->idx = idx;
3667         /*
3668          * The copy Flows are not included in any list. There
3669          * ones are referenced from other Flows and can not
3670          * be applied, removed, deleted in ardbitrary order
3671          * by list traversing.
3672          */
3673         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3674                                          actions, false, error);
3675         if (!mcp_res->rix_flow) {
3676                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
3677                 return NULL;
3678         }
3679         return &mcp_res->hlist_ent;
3680 }
3681
3682 /**
3683  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3684  *
3685  * As mark_id is unique, if there's already a registered flow for the mark_id,
3686  * return by increasing the reference counter of the resource. Otherwise, create
3687  * the resource (mcp_res) and flow.
3688  *
3689  * Flow looks like,
3690  *   - If ingress port is ANY and reg_c[1] is mark_id,
3691  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3692  *
3693  * For default flow (zero mark_id), flow is like,
3694  *   - If ingress port is ANY,
3695  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3696  *
3697  * @param dev
3698  *   Pointer to Ethernet device.
3699  * @param mark_id
3700  *   ID of MARK action, zero means default flow for META.
3701  * @param[out] error
3702  *   Perform verbose error reporting if not NULL.
3703  *
3704  * @return
3705  *   Associated resource on success, NULL otherwise and rte_errno is set.
3706  */
3707 static struct mlx5_flow_mreg_copy_resource *
3708 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3709                           struct rte_flow_error *error)
3710 {
3711         struct mlx5_priv *priv = dev->data->dev_private;
3712         struct mlx5_hlist_entry *entry;
3713         struct mlx5_flow_cb_ctx ctx = {
3714                 .dev = dev,
3715                 .error = error,
3716         };
3717
3718         /* Check if already registered. */
3719         MLX5_ASSERT(priv->mreg_cp_tbl);
3720         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
3721         if (!entry)
3722                 return NULL;
3723         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
3724                             hlist_ent);
3725 }
3726
3727 void
3728 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
3729 {
3730         struct mlx5_flow_mreg_copy_resource *mcp_res =
3731                 container_of(entry, typeof(*mcp_res), hlist_ent);
3732         struct rte_eth_dev *dev = list->ctx;
3733         struct mlx5_priv *priv = dev->data->dev_private;
3734
3735         MLX5_ASSERT(mcp_res->rix_flow);
3736         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3737         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3738 }
3739
3740 /**
3741  * Release flow in RX_CP_TBL.
3742  *
3743  * @param dev
3744  *   Pointer to Ethernet device.
3745  * @flow
3746  *   Parent flow for wich copying is provided.
3747  */
3748 static void
3749 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3750                           struct rte_flow *flow)
3751 {
3752         struct mlx5_flow_mreg_copy_resource *mcp_res;
3753         struct mlx5_priv *priv = dev->data->dev_private;
3754
3755         if (!flow->rix_mreg_copy)
3756                 return;
3757         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3758                                  flow->rix_mreg_copy);
3759         if (!mcp_res || !priv->mreg_cp_tbl)
3760                 return;
3761         MLX5_ASSERT(mcp_res->rix_flow);
3762         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3763         flow->rix_mreg_copy = 0;
3764 }
3765
3766 /**
3767  * Remove the default copy action from RX_CP_TBL.
3768  *
3769  * This functions is called in the mlx5_dev_start(). No thread safe
3770  * is guaranteed.
3771  *
3772  * @param dev
3773  *   Pointer to Ethernet device.
3774  */
3775 static void
3776 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3777 {
3778         struct mlx5_hlist_entry *entry;
3779         struct mlx5_priv *priv = dev->data->dev_private;
3780
3781         /* Check if default flow is registered. */
3782         if (!priv->mreg_cp_tbl)
3783                 return;
3784         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
3785                                   MLX5_DEFAULT_COPY_ID, NULL);
3786         if (!entry)
3787                 return;
3788         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
3789 }
3790
3791 /**
3792  * Add the default copy action in in RX_CP_TBL.
3793  *
3794  * This functions is called in the mlx5_dev_start(). No thread safe
3795  * is guaranteed.
3796  *
3797  * @param dev
3798  *   Pointer to Ethernet device.
3799  * @param[out] error
3800  *   Perform verbose error reporting if not NULL.
3801  *
3802  * @return
3803  *   0 for success, negative value otherwise and rte_errno is set.
3804  */
3805 static int
3806 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3807                                   struct rte_flow_error *error)
3808 {
3809         struct mlx5_priv *priv = dev->data->dev_private;
3810         struct mlx5_flow_mreg_copy_resource *mcp_res;
3811
3812         /* Check whether extensive metadata feature is engaged. */
3813         if (!priv->config.dv_flow_en ||
3814             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3815             !mlx5_flow_ext_mreg_supported(dev) ||
3816             !priv->sh->dv_regc0_mask)
3817                 return 0;
3818         /*
3819          * Add default mreg copy flow may be called multiple time, but
3820          * only be called once in stop. Avoid register it twice.
3821          */
3822         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
3823                 return 0;
3824         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3825         if (!mcp_res)
3826                 return -rte_errno;
3827         return 0;
3828 }
3829
3830 /**
3831  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3832  *
3833  * All the flow having Q/RSS action should be split by
3834  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3835  * performs the following,
3836  *   - CQE->flow_tag := reg_c[1] (MARK)
3837  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3838  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3839  * but there should be a flow per each MARK ID set by MARK action.
3840  *
3841  * For the aforementioned reason, if there's a MARK action in flow's action
3842  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3843  * the MARK ID to CQE's flow_tag like,
3844  *   - If reg_c[1] is mark_id,
3845  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3846  *
3847  * For SET_META action which stores value in reg_c[0], as the destination is
3848  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3849  * MARK ID means the default flow. The default flow looks like,
3850  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3851  *
3852  * @param dev
3853  *   Pointer to Ethernet device.
3854  * @param flow
3855  *   Pointer to flow structure.
3856  * @param[in] actions
3857  *   Pointer to the list of actions.
3858  * @param[out] error
3859  *   Perform verbose error reporting if not NULL.
3860  *
3861  * @return
3862  *   0 on success, negative value otherwise and rte_errno is set.
3863  */
3864 static int
3865 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3866                             struct rte_flow *flow,
3867                             const struct rte_flow_action *actions,
3868                             struct rte_flow_error *error)
3869 {
3870         struct mlx5_priv *priv = dev->data->dev_private;
3871         struct mlx5_dev_config *config = &priv->config;
3872         struct mlx5_flow_mreg_copy_resource *mcp_res;
3873         const struct rte_flow_action_mark *mark;
3874
3875         /* Check whether extensive metadata feature is engaged. */
3876         if (!config->dv_flow_en ||
3877             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3878             !mlx5_flow_ext_mreg_supported(dev) ||
3879             !priv->sh->dv_regc0_mask)
3880                 return 0;
3881         /* Find MARK action. */
3882         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3883                 switch (actions->type) {
3884                 case RTE_FLOW_ACTION_TYPE_FLAG:
3885                         mcp_res = flow_mreg_add_copy_action
3886                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3887                         if (!mcp_res)
3888                                 return -rte_errno;
3889                         flow->rix_mreg_copy = mcp_res->idx;
3890                         return 0;
3891                 case RTE_FLOW_ACTION_TYPE_MARK:
3892                         mark = (const struct rte_flow_action_mark *)
3893                                 actions->conf;
3894                         mcp_res =
3895                                 flow_mreg_add_copy_action(dev, mark->id, error);
3896                         if (!mcp_res)
3897                                 return -rte_errno;
3898                         flow->rix_mreg_copy = mcp_res->idx;
3899                         return 0;
3900                 default:
3901                         break;
3902                 }
3903         }
3904         return 0;
3905 }
3906
3907 #define MLX5_MAX_SPLIT_ACTIONS 24
3908 #define MLX5_MAX_SPLIT_ITEMS 24
3909
3910 /**
3911  * Split the hairpin flow.
3912  * Since HW can't support encap and push-vlan on Rx, we move these
3913  * actions to Tx.
3914  * If the count action is after the encap then we also
3915  * move the count action. in this case the count will also measure
3916  * the outer bytes.
3917  *
3918  * @param dev
3919  *   Pointer to Ethernet device.
3920  * @param[in] actions
3921  *   Associated actions (list terminated by the END action).
3922  * @param[out] actions_rx
3923  *   Rx flow actions.
3924  * @param[out] actions_tx
3925  *   Tx flow actions..
3926  * @param[out] pattern_tx
3927  *   The pattern items for the Tx flow.
3928  * @param[out] flow_id
3929  *   The flow ID connected to this flow.
3930  *
3931  * @return
3932  *   0 on success.
3933  */
3934 static int
3935 flow_hairpin_split(struct rte_eth_dev *dev,
3936                    const struct rte_flow_action actions[],
3937                    struct rte_flow_action actions_rx[],
3938                    struct rte_flow_action actions_tx[],
3939                    struct rte_flow_item pattern_tx[],
3940                    uint32_t flow_id)
3941 {
3942         const struct rte_flow_action_raw_encap *raw_encap;
3943         const struct rte_flow_action_raw_decap *raw_decap;
3944         struct mlx5_rte_flow_action_set_tag *set_tag;
3945         struct rte_flow_action *tag_action;
3946         struct mlx5_rte_flow_item_tag *tag_item;
3947         struct rte_flow_item *item;
3948         char *addr;
3949         int encap = 0;
3950
3951         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3952                 switch (actions->type) {
3953                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3954                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3955                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3956                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3957                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3958                         rte_memcpy(actions_tx, actions,
3959                                sizeof(struct rte_flow_action));
3960                         actions_tx++;
3961                         break;
3962                 case RTE_FLOW_ACTION_TYPE_COUNT:
3963                         if (encap) {
3964                                 rte_memcpy(actions_tx, actions,
3965                                            sizeof(struct rte_flow_action));
3966                                 actions_tx++;
3967                         } else {
3968                                 rte_memcpy(actions_rx, actions,
3969                                            sizeof(struct rte_flow_action));
3970                                 actions_rx++;
3971                         }
3972                         break;
3973                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3974                         raw_encap = actions->conf;
3975                         if (raw_encap->size >
3976                             (sizeof(struct rte_flow_item_eth) +
3977                              sizeof(struct rte_flow_item_ipv4))) {
3978                                 memcpy(actions_tx, actions,
3979                                        sizeof(struct rte_flow_action));
3980                                 actions_tx++;
3981                                 encap = 1;
3982                         } else {
3983                                 rte_memcpy(actions_rx, actions,
3984                                            sizeof(struct rte_flow_action));
3985                                 actions_rx++;
3986                         }
3987                         break;
3988                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3989                         raw_decap = actions->conf;
3990                         if (raw_decap->size <
3991                             (sizeof(struct rte_flow_item_eth) +
3992                              sizeof(struct rte_flow_item_ipv4))) {
3993                                 memcpy(actions_tx, actions,
3994                                        sizeof(struct rte_flow_action));
3995                                 actions_tx++;
3996                         } else {
3997                                 rte_memcpy(actions_rx, actions,
3998                                            sizeof(struct rte_flow_action));
3999                                 actions_rx++;
4000                         }
4001                         break;
4002                 default:
4003                         rte_memcpy(actions_rx, actions,
4004                                    sizeof(struct rte_flow_action));
4005                         actions_rx++;
4006                         break;
4007                 }
4008         }
4009         /* Add set meta action and end action for the Rx flow. */
4010         tag_action = actions_rx;
4011         tag_action->type = (enum rte_flow_action_type)
4012                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4013         actions_rx++;
4014         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4015         actions_rx++;
4016         set_tag = (void *)actions_rx;
4017         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4018         MLX5_ASSERT(set_tag->id > REG_NON);
4019         set_tag->data = flow_id;
4020         tag_action->conf = set_tag;
4021         /* Create Tx item list. */
4022         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4023         addr = (void *)&pattern_tx[2];
4024         item = pattern_tx;
4025         item->type = (enum rte_flow_item_type)
4026                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4027         tag_item = (void *)addr;
4028         tag_item->data = flow_id;
4029         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4030         MLX5_ASSERT(set_tag->id > REG_NON);
4031         item->spec = tag_item;
4032         addr += sizeof(struct mlx5_rte_flow_item_tag);
4033         tag_item = (void *)addr;
4034         tag_item->data = UINT32_MAX;
4035         tag_item->id = UINT16_MAX;
4036         item->mask = tag_item;
4037         item->last = NULL;
4038         item++;
4039         item->type = RTE_FLOW_ITEM_TYPE_END;
4040         return 0;
4041 }
4042
4043 /**
4044  * The last stage of splitting chain, just creates the subflow
4045  * without any modification.
4046  *
4047  * @param[in] dev
4048  *   Pointer to Ethernet device.
4049  * @param[in] flow
4050  *   Parent flow structure pointer.
4051  * @param[in, out] sub_flow
4052  *   Pointer to return the created subflow, may be NULL.
4053  * @param[in] attr
4054  *   Flow rule attributes.
4055  * @param[in] items
4056  *   Pattern specification (list terminated by the END pattern item).
4057  * @param[in] actions
4058  *   Associated actions (list terminated by the END action).
4059  * @param[in] flow_split_info
4060  *   Pointer to flow split info structure.
4061  * @param[out] error
4062  *   Perform verbose error reporting if not NULL.
4063  * @return
4064  *   0 on success, negative value otherwise
4065  */
4066 static int
4067 flow_create_split_inner(struct rte_eth_dev *dev,
4068                         struct rte_flow *flow,
4069                         struct mlx5_flow **sub_flow,
4070                         const struct rte_flow_attr *attr,
4071                         const struct rte_flow_item items[],
4072                         const struct rte_flow_action actions[],
4073                         struct mlx5_flow_split_info *flow_split_info,
4074                         struct rte_flow_error *error)
4075 {
4076         struct mlx5_flow *dev_flow;
4077
4078         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4079                                     flow_split_info->flow_idx, error);
4080         if (!dev_flow)
4081                 return -rte_errno;
4082         dev_flow->flow = flow;
4083         dev_flow->external = flow_split_info->external;
4084         dev_flow->skip_scale = flow_split_info->skip_scale;
4085         /* Subflow object was created, we must include one in the list. */
4086         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4087                       dev_flow->handle, next);
4088         /*
4089          * If dev_flow is as one of the suffix flow, some actions in suffix
4090          * flow may need some user defined item layer flags, and pass the
4091          * Metadate rxq mark flag to suffix flow as well.
4092          */
4093         if (flow_split_info->prefix_layers)
4094                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4095         if (flow_split_info->prefix_mark)
4096                 dev_flow->handle->mark = 1;
4097         if (sub_flow)
4098                 *sub_flow = dev_flow;
4099         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4100 }
4101
4102 /**
4103  * Split the meter flow.
4104  *
4105  * As meter flow will split to three sub flow, other than meter
4106  * action, the other actions make sense to only meter accepts
4107  * the packet. If it need to be dropped, no other additional
4108  * actions should be take.
4109  *
4110  * One kind of special action which decapsulates the L3 tunnel
4111  * header will be in the prefix sub flow, as not to take the
4112  * L3 tunnel header into account.
4113  *
4114  * @param dev
4115  *   Pointer to Ethernet device.
4116  * @param[in] items
4117  *   Pattern specification (list terminated by the END pattern item).
4118  * @param[out] sfx_items
4119  *   Suffix flow match items (list terminated by the END pattern item).
4120  * @param[in] actions
4121  *   Associated actions (list terminated by the END action).
4122  * @param[out] actions_sfx
4123  *   Suffix flow actions.
4124  * @param[out] actions_pre
4125  *   Prefix flow actions.
4126  * @param[out] pattern_sfx
4127  *   The pattern items for the suffix flow.
4128  * @param[out] tag_sfx
4129  *   Pointer to suffix flow tag.
4130  *
4131  * @return
4132  *   0 on success.
4133  */
4134 static int
4135 flow_meter_split_prep(struct rte_eth_dev *dev,
4136                  const struct rte_flow_item items[],
4137                  struct rte_flow_item sfx_items[],
4138                  const struct rte_flow_action actions[],
4139                  struct rte_flow_action actions_sfx[],
4140                  struct rte_flow_action actions_pre[])
4141 {
4142         struct mlx5_priv *priv = dev->data->dev_private;
4143         struct rte_flow_action *tag_action = NULL;
4144         struct rte_flow_item *tag_item;
4145         struct mlx5_rte_flow_action_set_tag *set_tag;
4146         struct rte_flow_error error;
4147         const struct rte_flow_action_raw_encap *raw_encap;
4148         const struct rte_flow_action_raw_decap *raw_decap;
4149         struct mlx5_rte_flow_item_tag *tag_spec;
4150         struct mlx5_rte_flow_item_tag *tag_mask;
4151         uint32_t tag_id = 0;
4152         bool copy_vlan = false;
4153
4154         /* Prepare the actions for prefix and suffix flow. */
4155         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4156                 struct rte_flow_action **action_cur = NULL;
4157
4158                 switch (actions->type) {
4159                 case RTE_FLOW_ACTION_TYPE_METER:
4160                         /* Add the extra tag action first. */
4161                         tag_action = actions_pre;
4162                         tag_action->type = (enum rte_flow_action_type)
4163                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4164                         actions_pre++;
4165                         action_cur = &actions_pre;
4166                         break;
4167                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4168                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4169                         action_cur = &actions_pre;
4170                         break;
4171                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4172                         raw_encap = actions->conf;
4173                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4174                                 action_cur = &actions_pre;
4175                         break;
4176                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4177                         raw_decap = actions->conf;
4178                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4179                                 action_cur = &actions_pre;
4180                         break;
4181                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4182                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4183                         copy_vlan = true;
4184                         break;
4185                 default:
4186                         break;
4187                 }
4188                 if (!action_cur)
4189                         action_cur = &actions_sfx;
4190                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4191                 (*action_cur)++;
4192         }
4193         /* Add end action to the actions. */
4194         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4195         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4196         actions_pre++;
4197         /* Set the tag. */
4198         set_tag = (void *)actions_pre;
4199         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4200         mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4201                           &tag_id);
4202         if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
4203                 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
4204                         dev->data->port_id);
4205                 mlx5_ipool_free(priv->sh->ipool
4206                                 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
4207                 return 0;
4208         } else if (!tag_id) {
4209                 return 0;
4210         }
4211         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4212         assert(tag_action);
4213         tag_action->conf = set_tag;
4214         /* Prepare the suffix subflow items. */
4215         tag_item = sfx_items++;
4216         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4217                 int item_type = items->type;
4218
4219                 switch (item_type) {
4220                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4221                         memcpy(sfx_items, items, sizeof(*sfx_items));
4222                         sfx_items++;
4223                         break;
4224                 case RTE_FLOW_ITEM_TYPE_VLAN:
4225                         if (copy_vlan) {
4226                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4227                                 /*
4228                                  * Convert to internal match item, it is used
4229                                  * for vlan push and set vid.
4230                                  */
4231                                 sfx_items->type = (enum rte_flow_item_type)
4232                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4233                                 sfx_items++;
4234                         }
4235                         break;
4236                 default:
4237                         break;
4238                 }
4239         }
4240         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4241         sfx_items++;
4242         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4243         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4244         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4245         tag_mask = tag_spec + 1;
4246         tag_mask->data = 0xffffff00;
4247         tag_item->type = (enum rte_flow_item_type)
4248                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4249         tag_item->spec = tag_spec;
4250         tag_item->last = NULL;
4251         tag_item->mask = tag_mask;
4252         return tag_id;
4253 }
4254
4255 /**
4256  * Split action list having QUEUE/RSS for metadata register copy.
4257  *
4258  * Once Q/RSS action is detected in user's action list, the flow action
4259  * should be split in order to copy metadata registers, which will happen in
4260  * RX_CP_TBL like,
4261  *   - CQE->flow_tag := reg_c[1] (MARK)
4262  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4263  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4264  * This is because the last action of each flow must be a terminal action
4265  * (QUEUE, RSS or DROP).
4266  *
4267  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4268  * stored and kept in the mlx5_flow structure per each sub_flow.
4269  *
4270  * The Q/RSS action is replaced with,
4271  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4272  * And the following JUMP action is added at the end,
4273  *   - JUMP, to RX_CP_TBL.
4274  *
4275  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4276  * flow_create_split_metadata() routine. The flow will look like,
4277  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4278  *
4279  * @param dev
4280  *   Pointer to Ethernet device.
4281  * @param[out] split_actions
4282  *   Pointer to store split actions to jump to CP_TBL.
4283  * @param[in] actions
4284  *   Pointer to the list of original flow actions.
4285  * @param[in] qrss
4286  *   Pointer to the Q/RSS action.
4287  * @param[in] actions_n
4288  *   Number of original actions.
4289  * @param[out] error
4290  *   Perform verbose error reporting if not NULL.
4291  *
4292  * @return
4293  *   non-zero unique flow_id on success, otherwise 0 and
4294  *   error/rte_error are set.
4295  */
4296 static uint32_t
4297 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4298                           struct rte_flow_action *split_actions,
4299                           const struct rte_flow_action *actions,
4300                           const struct rte_flow_action *qrss,
4301                           int actions_n, struct rte_flow_error *error)
4302 {
4303         struct mlx5_priv *priv = dev->data->dev_private;
4304         struct mlx5_rte_flow_action_set_tag *set_tag;
4305         struct rte_flow_action_jump *jump;
4306         const int qrss_idx = qrss - actions;
4307         uint32_t flow_id = 0;
4308         int ret = 0;
4309
4310         /*
4311          * Given actions will be split
4312          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4313          * - Add jump to mreg CP_TBL.
4314          * As a result, there will be one more action.
4315          */
4316         ++actions_n;
4317         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4318         set_tag = (void *)(split_actions + actions_n);
4319         /*
4320          * If tag action is not set to void(it means we are not the meter
4321          * suffix flow), add the tag action. Since meter suffix flow already
4322          * has the tag added.
4323          */
4324         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4325                 /*
4326                  * Allocate the new subflow ID. This one is unique within
4327                  * device and not shared with representors. Otherwise,
4328                  * we would have to resolve multi-thread access synch
4329                  * issue. Each flow on the shared device is appended
4330                  * with source vport identifier, so the resulting
4331                  * flows will be unique in the shared (by master and
4332                  * representors) domain even if they have coinciding
4333                  * IDs.
4334                  */
4335                 mlx5_ipool_malloc(priv->sh->ipool
4336                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4337                 if (!flow_id)
4338                         return rte_flow_error_set(error, ENOMEM,
4339                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4340                                                   NULL, "can't allocate id "
4341                                                   "for split Q/RSS subflow");
4342                 /* Internal SET_TAG action to set flow ID. */
4343                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4344                         .data = flow_id,
4345                 };
4346                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4347                 if (ret < 0)
4348                         return ret;
4349                 set_tag->id = ret;
4350                 /* Construct new actions array. */
4351                 /* Replace QUEUE/RSS action. */
4352                 split_actions[qrss_idx] = (struct rte_flow_action){
4353                         .type = (enum rte_flow_action_type)
4354                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4355                         .conf = set_tag,
4356                 };
4357         }
4358         /* JUMP action to jump to mreg copy table (CP_TBL). */
4359         jump = (void *)(set_tag + 1);
4360         *jump = (struct rte_flow_action_jump){
4361                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4362         };
4363         split_actions[actions_n - 2] = (struct rte_flow_action){
4364                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4365                 .conf = jump,
4366         };
4367         split_actions[actions_n - 1] = (struct rte_flow_action){
4368                 .type = RTE_FLOW_ACTION_TYPE_END,
4369         };
4370         return flow_id;
4371 }
4372
4373 /**
4374  * Extend the given action list for Tx metadata copy.
4375  *
4376  * Copy the given action list to the ext_actions and add flow metadata register
4377  * copy action in order to copy reg_a set by WQE to reg_c[0].
4378  *
4379  * @param[out] ext_actions
4380  *   Pointer to the extended action list.
4381  * @param[in] actions
4382  *   Pointer to the list of actions.
4383  * @param[in] actions_n
4384  *   Number of actions in the list.
4385  * @param[out] error
4386  *   Perform verbose error reporting if not NULL.
4387  * @param[in] encap_idx
4388  *   The encap action inndex.
4389  *
4390  * @return
4391  *   0 on success, negative value otherwise
4392  */
4393 static int
4394 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4395                        struct rte_flow_action *ext_actions,
4396                        const struct rte_flow_action *actions,
4397                        int actions_n, struct rte_flow_error *error,
4398                        int encap_idx)
4399 {
4400         struct mlx5_flow_action_copy_mreg *cp_mreg =
4401                 (struct mlx5_flow_action_copy_mreg *)
4402                         (ext_actions + actions_n + 1);
4403         int ret;
4404
4405         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4406         if (ret < 0)
4407                 return ret;
4408         cp_mreg->dst = ret;
4409         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4410         if (ret < 0)
4411                 return ret;
4412         cp_mreg->src = ret;
4413         if (encap_idx != 0)
4414                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4415         if (encap_idx == actions_n - 1) {
4416                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4417                         .type = (enum rte_flow_action_type)
4418                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4419                         .conf = cp_mreg,
4420                 };
4421                 ext_actions[actions_n] = (struct rte_flow_action){
4422                         .type = RTE_FLOW_ACTION_TYPE_END,
4423                 };
4424         } else {
4425                 ext_actions[encap_idx] = (struct rte_flow_action){
4426                         .type = (enum rte_flow_action_type)
4427                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4428                         .conf = cp_mreg,
4429                 };
4430                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4431                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4432         }
4433         return 0;
4434 }
4435
4436 /**
4437  * Check the match action from the action list.
4438  *
4439  * @param[in] actions
4440  *   Pointer to the list of actions.
4441  * @param[in] attr
4442  *   Flow rule attributes.
4443  * @param[in] action
4444  *   The action to be check if exist.
4445  * @param[out] match_action_pos
4446  *   Pointer to the position of the matched action if exists, otherwise is -1.
4447  * @param[out] qrss_action_pos
4448  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4449  *
4450  * @return
4451  *   > 0 the total number of actions.
4452  *   0 if not found match action in action list.
4453  */
4454 static int
4455 flow_check_match_action(const struct rte_flow_action actions[],
4456                         const struct rte_flow_attr *attr,
4457                         enum rte_flow_action_type action,
4458                         int *match_action_pos, int *qrss_action_pos)
4459 {
4460         const struct rte_flow_action_sample *sample;
4461         int actions_n = 0;
4462         int jump_flag = 0;
4463         uint32_t ratio = 0;
4464         int sub_type = 0;
4465         int flag = 0;
4466
4467         *match_action_pos = -1;
4468         *qrss_action_pos = -1;
4469         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4470                 if (actions->type == action) {
4471                         flag = 1;
4472                         *match_action_pos = actions_n;
4473                 }
4474                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4475                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4476                         *qrss_action_pos = actions_n;
4477                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4478                         jump_flag = 1;
4479                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4480                         sample = actions->conf;
4481                         ratio = sample->ratio;
4482                         sub_type = ((const struct rte_flow_action *)
4483                                         (sample->actions))->type;
4484                 }
4485                 actions_n++;
4486         }
4487         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4488                 if (ratio == 1) {
4489                         /* JUMP Action not support for Mirroring;
4490                          * Mirroring support multi-destination;
4491                          */
4492                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4493                                 flag = 0;
4494                 }
4495         }
4496         /* Count RTE_FLOW_ACTION_TYPE_END. */
4497         return flag ? actions_n + 1 : 0;
4498 }
4499
4500 #define SAMPLE_SUFFIX_ITEM 2
4501
4502 /**
4503  * Split the sample flow.
4504  *
4505  * As sample flow will split to two sub flow, sample flow with
4506  * sample action, the other actions will move to new suffix flow.
4507  *
4508  * Also add unique tag id with tag action in the sample flow,
4509  * the same tag id will be as match in the suffix flow.
4510  *
4511  * @param dev
4512  *   Pointer to Ethernet device.
4513  * @param[in] fdb_tx
4514  *   FDB egress flow flag.
4515  * @param[out] sfx_items
4516  *   Suffix flow match items (list terminated by the END pattern item).
4517  * @param[in] actions
4518  *   Associated actions (list terminated by the END action).
4519  * @param[out] actions_sfx
4520  *   Suffix flow actions.
4521  * @param[out] actions_pre
4522  *   Prefix flow actions.
4523  * @param[in] actions_n
4524  *  The total number of actions.
4525  * @param[in] sample_action_pos
4526  *   The sample action position.
4527  * @param[in] qrss_action_pos
4528  *   The Queue/RSS action position.
4529  * @param[out] error
4530  *   Perform verbose error reporting if not NULL.
4531  *
4532  * @return
4533  *   0 on success, or unique flow_id, a negative errno value
4534  *   otherwise and rte_errno is set.
4535  */
4536 static int
4537 flow_sample_split_prep(struct rte_eth_dev *dev,
4538                        uint32_t fdb_tx,
4539                        struct rte_flow_item sfx_items[],
4540                        const struct rte_flow_action actions[],
4541                        struct rte_flow_action actions_sfx[],
4542                        struct rte_flow_action actions_pre[],
4543                        int actions_n,
4544                        int sample_action_pos,
4545                        int qrss_action_pos,
4546                        struct rte_flow_error *error)
4547 {
4548         struct mlx5_priv *priv = dev->data->dev_private;
4549         struct mlx5_rte_flow_action_set_tag *set_tag;
4550         struct mlx5_rte_flow_item_tag *tag_spec;
4551         struct mlx5_rte_flow_item_tag *tag_mask;
4552         uint32_t tag_id = 0;
4553         int index;
4554         int ret;
4555
4556         if (sample_action_pos < 0)
4557                 return rte_flow_error_set(error, EINVAL,
4558                                           RTE_FLOW_ERROR_TYPE_ACTION,
4559                                           NULL, "invalid position of sample "
4560                                           "action in list");
4561         if (!fdb_tx) {
4562                 /* Prepare the prefix tag action. */
4563                 set_tag = (void *)(actions_pre + actions_n + 1);
4564                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4565                 if (ret < 0)
4566                         return ret;
4567                 set_tag->id = ret;
4568                 mlx5_ipool_malloc(priv->sh->ipool
4569                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
4570                 set_tag->data = tag_id;
4571                 /* Prepare the suffix subflow items. */
4572                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4573                 tag_spec->data = tag_id;
4574                 tag_spec->id = set_tag->id;
4575                 tag_mask = tag_spec + 1;
4576                 tag_mask->data = UINT32_MAX;
4577                 sfx_items[0] = (struct rte_flow_item){
4578                         .type = (enum rte_flow_item_type)
4579                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4580                         .spec = tag_spec,
4581                         .last = NULL,
4582                         .mask = tag_mask,
4583                 };
4584                 sfx_items[1] = (struct rte_flow_item){
4585                         .type = (enum rte_flow_item_type)
4586                                 RTE_FLOW_ITEM_TYPE_END,
4587                 };
4588         }
4589         /* Prepare the actions for prefix and suffix flow. */
4590         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4591                 index = qrss_action_pos;
4592                 /* Put the preceding the Queue/RSS action into prefix flow. */
4593                 if (index != 0)
4594                         memcpy(actions_pre, actions,
4595                                sizeof(struct rte_flow_action) * index);
4596                 /* Put others preceding the sample action into prefix flow. */
4597                 if (sample_action_pos > index + 1)
4598                         memcpy(actions_pre + index, actions + index + 1,
4599                                sizeof(struct rte_flow_action) *
4600                                (sample_action_pos - index - 1));
4601                 index = sample_action_pos - 1;
4602                 /* Put Queue/RSS action into Suffix flow. */
4603                 memcpy(actions_sfx, actions + qrss_action_pos,
4604                        sizeof(struct rte_flow_action));
4605                 actions_sfx++;
4606         } else {
4607                 index = sample_action_pos;
4608                 if (index != 0)
4609                         memcpy(actions_pre, actions,
4610                                sizeof(struct rte_flow_action) * index);
4611         }
4612         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4613         if (!fdb_tx) {
4614                 actions_pre[index++] =
4615                         (struct rte_flow_action){
4616                         .type = (enum rte_flow_action_type)
4617                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4618                         .conf = set_tag,
4619                 };
4620         }
4621         memcpy(actions_pre + index, actions + sample_action_pos,
4622                sizeof(struct rte_flow_action));
4623         index += 1;
4624         actions_pre[index] = (struct rte_flow_action){
4625                 .type = (enum rte_flow_action_type)
4626                         RTE_FLOW_ACTION_TYPE_END,
4627         };
4628         /* Put the actions after sample into Suffix flow. */
4629         memcpy(actions_sfx, actions + sample_action_pos + 1,
4630                sizeof(struct rte_flow_action) *
4631                (actions_n - sample_action_pos - 1));
4632         return tag_id;
4633 }
4634
4635 /**
4636  * The splitting for metadata feature.
4637  *
4638  * - Q/RSS action on NIC Rx should be split in order to pass by
4639  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4640  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4641  *
4642  * - All the actions on NIC Tx should have a mreg copy action to
4643  *   copy reg_a from WQE to reg_c[0].
4644  *
4645  * @param dev
4646  *   Pointer to Ethernet device.
4647  * @param[in] flow
4648  *   Parent flow structure pointer.
4649  * @param[in] attr
4650  *   Flow rule attributes.
4651  * @param[in] items
4652  *   Pattern specification (list terminated by the END pattern item).
4653  * @param[in] actions
4654  *   Associated actions (list terminated by the END action).
4655  * @param[in] flow_split_info
4656  *   Pointer to flow split info structure.
4657  * @param[out] error
4658  *   Perform verbose error reporting if not NULL.
4659  * @return
4660  *   0 on success, negative value otherwise
4661  */
4662 static int
4663 flow_create_split_metadata(struct rte_eth_dev *dev,
4664                            struct rte_flow *flow,
4665                            const struct rte_flow_attr *attr,
4666                            const struct rte_flow_item items[],
4667                            const struct rte_flow_action actions[],
4668                            struct mlx5_flow_split_info *flow_split_info,
4669                            struct rte_flow_error *error)
4670 {
4671         struct mlx5_priv *priv = dev->data->dev_private;
4672         struct mlx5_dev_config *config = &priv->config;
4673         const struct rte_flow_action *qrss = NULL;
4674         struct rte_flow_action *ext_actions = NULL;
4675         struct mlx5_flow *dev_flow = NULL;
4676         uint32_t qrss_id = 0;
4677         int mtr_sfx = 0;
4678         size_t act_size;
4679         int actions_n;
4680         int encap_idx;
4681         int ret;
4682
4683         /* Check whether extensive metadata feature is engaged. */
4684         if (!config->dv_flow_en ||
4685             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4686             !mlx5_flow_ext_mreg_supported(dev))
4687                 return flow_create_split_inner(dev, flow, NULL, attr, items,
4688                                                actions, flow_split_info, error);
4689         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4690                                                            &encap_idx);
4691         if (qrss) {
4692                 /* Exclude hairpin flows from splitting. */
4693                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4694                         const struct rte_flow_action_queue *queue;
4695
4696                         queue = qrss->conf;
4697                         if (mlx5_rxq_get_type(dev, queue->index) ==
4698                             MLX5_RXQ_TYPE_HAIRPIN)
4699                                 qrss = NULL;
4700                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4701                         const struct rte_flow_action_rss *rss;
4702
4703                         rss = qrss->conf;
4704                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4705                             MLX5_RXQ_TYPE_HAIRPIN)
4706                                 qrss = NULL;
4707                 }
4708         }
4709         if (qrss) {
4710                 /* Check if it is in meter suffix table. */
4711                 mtr_sfx = attr->group == (attr->transfer ?
4712                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4713                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4714                 /*
4715                  * Q/RSS action on NIC Rx should be split in order to pass by
4716                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4717                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4718                  */
4719                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4720                            sizeof(struct rte_flow_action_set_tag) +
4721                            sizeof(struct rte_flow_action_jump);
4722                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4723                                           SOCKET_ID_ANY);
4724                 if (!ext_actions)
4725                         return rte_flow_error_set(error, ENOMEM,
4726                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4727                                                   NULL, "no memory to split "
4728                                                   "metadata flow");
4729                 /*
4730                  * If we are the suffix flow of meter, tag already exist.
4731                  * Set the tag action to void.
4732                  */
4733                 if (mtr_sfx)
4734                         ext_actions[qrss - actions].type =
4735                                                 RTE_FLOW_ACTION_TYPE_VOID;
4736                 else
4737                         ext_actions[qrss - actions].type =
4738                                                 (enum rte_flow_action_type)
4739                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4740                 /*
4741                  * Create the new actions list with removed Q/RSS action
4742                  * and appended set tag and jump to register copy table
4743                  * (RX_CP_TBL). We should preallocate unique tag ID here
4744                  * in advance, because it is needed for set tag action.
4745                  */
4746                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4747                                                     qrss, actions_n, error);
4748                 if (!mtr_sfx && !qrss_id) {
4749                         ret = -rte_errno;
4750                         goto exit;
4751                 }
4752         } else if (attr->egress && !attr->transfer) {
4753                 /*
4754                  * All the actions on NIC Tx should have a metadata register
4755                  * copy action to copy reg_a from WQE to reg_c[meta]
4756                  */
4757                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4758                            sizeof(struct mlx5_flow_action_copy_mreg);
4759                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4760                                           SOCKET_ID_ANY);
4761                 if (!ext_actions)
4762                         return rte_flow_error_set(error, ENOMEM,
4763                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4764                                                   NULL, "no memory to split "
4765                                                   "metadata flow");
4766                 /* Create the action list appended with copy register. */
4767                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4768                                              actions_n, error, encap_idx);
4769                 if (ret < 0)
4770                         goto exit;
4771         }
4772         /* Add the unmodified original or prefix subflow. */
4773         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
4774                                       items, ext_actions ? ext_actions :
4775                                       actions, flow_split_info, error);
4776         if (ret < 0)
4777                 goto exit;
4778         MLX5_ASSERT(dev_flow);
4779         if (qrss) {
4780                 const struct rte_flow_attr q_attr = {
4781                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4782                         .ingress = 1,
4783                 };
4784                 /* Internal PMD action to set register. */
4785                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4786                         .data = qrss_id,
4787                         .id = REG_NON,
4788                 };
4789                 struct rte_flow_item q_items[] = {
4790                         {
4791                                 .type = (enum rte_flow_item_type)
4792                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4793                                 .spec = &q_tag_spec,
4794                                 .last = NULL,
4795                                 .mask = NULL,
4796                         },
4797                         {
4798                                 .type = RTE_FLOW_ITEM_TYPE_END,
4799                         },
4800                 };
4801                 struct rte_flow_action q_actions[] = {
4802                         {
4803                                 .type = qrss->type,
4804                                 .conf = qrss->conf,
4805                         },
4806                         {
4807                                 .type = RTE_FLOW_ACTION_TYPE_END,
4808                         },
4809                 };
4810                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4811
4812                 /*
4813                  * Configure the tag item only if there is no meter subflow.
4814                  * Since tag is already marked in the meter suffix subflow
4815                  * we can just use the meter suffix items as is.
4816                  */
4817                 if (qrss_id) {
4818                         /* Not meter subflow. */
4819                         MLX5_ASSERT(!mtr_sfx);
4820                         /*
4821                          * Put unique id in prefix flow due to it is destroyed
4822                          * after suffix flow and id will be freed after there
4823                          * is no actual flows with this id and identifier
4824                          * reallocation becomes possible (for example, for
4825                          * other flows in other threads).
4826                          */
4827                         dev_flow->handle->split_flow_id = qrss_id;
4828                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4829                                                    error);
4830                         if (ret < 0)
4831                                 goto exit;
4832                         q_tag_spec.id = ret;
4833                 }
4834                 dev_flow = NULL;
4835                 /* Add suffix subflow to execute Q/RSS. */
4836                 flow_split_info->prefix_layers = layers;
4837                 flow_split_info->prefix_mark = 0;
4838                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4839                                               &q_attr, mtr_sfx ? items :
4840                                               q_items, q_actions,
4841                                               flow_split_info, error);
4842                 if (ret < 0)
4843                         goto exit;
4844                 /* qrss ID should be freed if failed. */
4845                 qrss_id = 0;
4846                 MLX5_ASSERT(dev_flow);
4847         }
4848
4849 exit:
4850         /*
4851          * We do not destroy the partially created sub_flows in case of error.
4852          * These ones are included into parent flow list and will be destroyed
4853          * by flow_drv_destroy.
4854          */
4855         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4856                         qrss_id);
4857         mlx5_free(ext_actions);
4858         return ret;
4859 }
4860
4861 /**
4862  * The splitting for meter feature.
4863  *
4864  * - The meter flow will be split to two flows as prefix and
4865  *   suffix flow. The packets make sense only it pass the prefix
4866  *   meter action.
4867  *
4868  * - Reg_C_5 is used for the packet to match betweend prefix and
4869  *   suffix flow.
4870  *
4871  * @param dev
4872  *   Pointer to Ethernet device.
4873  * @param[in] flow
4874  *   Parent flow structure pointer.
4875  * @param[in] attr
4876  *   Flow rule attributes.
4877  * @param[in] items
4878  *   Pattern specification (list terminated by the END pattern item).
4879  * @param[in] actions
4880  *   Associated actions (list terminated by the END action).
4881  * @param[in] flow_split_info
4882  *   Pointer to flow split info structure.
4883  * @param[out] error
4884  *   Perform verbose error reporting if not NULL.
4885  * @return
4886  *   0 on success, negative value otherwise
4887  */
4888 static int
4889 flow_create_split_meter(struct rte_eth_dev *dev,
4890                         struct rte_flow *flow,
4891                         const struct rte_flow_attr *attr,
4892                         const struct rte_flow_item items[],
4893                         const struct rte_flow_action actions[],
4894                         struct mlx5_flow_split_info *flow_split_info,
4895                         struct rte_flow_error *error)
4896 {
4897         struct mlx5_priv *priv = dev->data->dev_private;
4898         struct rte_flow_action *sfx_actions = NULL;
4899         struct rte_flow_action *pre_actions = NULL;
4900         struct rte_flow_item *sfx_items = NULL;
4901         struct mlx5_flow *dev_flow = NULL;
4902         struct rte_flow_attr sfx_attr = *attr;
4903         uint32_t mtr = 0;
4904         uint32_t mtr_tag_id = 0;
4905         size_t act_size;
4906         size_t item_size;
4907         int actions_n = 0;
4908         int ret;
4909
4910         if (priv->mtr_en)
4911                 actions_n = flow_check_meter_action(actions, &mtr);
4912         if (mtr) {
4913                 /* The five prefix actions: meter, decap, encap, tag, end. */
4914                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4915                            sizeof(struct mlx5_rte_flow_action_set_tag);
4916                 /* tag, vlan, port id, end. */
4917 #define METER_SUFFIX_ITEM 4
4918                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4919                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4920                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4921                                           0, SOCKET_ID_ANY);
4922                 if (!sfx_actions)
4923                         return rte_flow_error_set(error, ENOMEM,
4924                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4925                                                   NULL, "no memory to split "
4926                                                   "meter flow");
4927                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4928                              act_size);
4929                 pre_actions = sfx_actions + actions_n;
4930                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4931                                                    actions, sfx_actions,
4932                                                    pre_actions);
4933                 if (!mtr_tag_id) {
4934                         ret = -rte_errno;
4935                         goto exit;
4936                 }
4937                 /* Add the prefix subflow. */
4938                 flow_split_info->prefix_mark = 0;
4939                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4940                                               attr, items, pre_actions,
4941                                               flow_split_info, error);
4942                 if (ret) {
4943                         ret = -rte_errno;
4944                         goto exit;
4945                 }
4946                 dev_flow->handle->split_flow_id = mtr_tag_id;
4947                 /* Setting the sfx group atrr. */
4948                 sfx_attr.group = sfx_attr.transfer ?
4949                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4950                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4951                 flow_split_info->prefix_layers =
4952                                 flow_get_prefix_layer_flags(dev_flow);
4953                 flow_split_info->prefix_mark = dev_flow->handle->mark;
4954         }
4955         /* Add the prefix subflow. */
4956         ret = flow_create_split_metadata(dev, flow,
4957                                          &sfx_attr, sfx_items ?
4958                                          sfx_items : items,
4959                                          sfx_actions ? sfx_actions : actions,
4960                                          flow_split_info, error);
4961 exit:
4962         if (sfx_actions)
4963                 mlx5_free(sfx_actions);
4964         return ret;
4965 }
4966
4967 /**
4968  * The splitting for sample feature.
4969  *
4970  * Once Sample action is detected in the action list, the flow actions should
4971  * be split into prefix sub flow and suffix sub flow.
4972  *
4973  * The original items remain in the prefix sub flow, all actions preceding the
4974  * sample action and the sample action itself will be copied to the prefix
4975  * sub flow, the actions following the sample action will be copied to the
4976  * suffix sub flow, Queue action always be located in the suffix sub flow.
4977  *
4978  * In order to make the packet from prefix sub flow matches with suffix sub
4979  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
4980  * flow uses tag item with the unique flow id.
4981  *
4982  * @param dev
4983  *   Pointer to Ethernet device.
4984  * @param[in] flow
4985  *   Parent flow structure pointer.
4986  * @param[in] attr
4987  *   Flow rule attributes.
4988  * @param[in] items
4989  *   Pattern specification (list terminated by the END pattern item).
4990  * @param[in] actions
4991  *   Associated actions (list terminated by the END action).
4992  * @param[in] flow_split_info
4993  *   Pointer to flow split info structure.
4994  * @param[out] error
4995  *   Perform verbose error reporting if not NULL.
4996  * @return
4997  *   0 on success, negative value otherwise
4998  */
4999 static int
5000 flow_create_split_sample(struct rte_eth_dev *dev,
5001                          struct rte_flow *flow,
5002                          const struct rte_flow_attr *attr,
5003                          const struct rte_flow_item items[],
5004                          const struct rte_flow_action actions[],
5005                          struct mlx5_flow_split_info *flow_split_info,
5006                          struct rte_flow_error *error)
5007 {
5008         struct mlx5_priv *priv = dev->data->dev_private;
5009         struct rte_flow_action *sfx_actions = NULL;
5010         struct rte_flow_action *pre_actions = NULL;
5011         struct rte_flow_item *sfx_items = NULL;
5012         struct mlx5_flow *dev_flow = NULL;
5013         struct rte_flow_attr sfx_attr = *attr;
5014 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5015         struct mlx5_flow_dv_sample_resource *sample_res;
5016         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5017         struct mlx5_flow_tbl_resource *sfx_tbl;
5018         union mlx5_flow_tbl_key sfx_table_key;
5019 #endif
5020         size_t act_size;
5021         size_t item_size;
5022         uint32_t fdb_tx = 0;
5023         int32_t tag_id = 0;
5024         int actions_n = 0;
5025         int sample_action_pos;
5026         int qrss_action_pos;
5027         int ret = 0;
5028
5029         if (priv->sampler_en)
5030                 actions_n = flow_check_match_action(actions, attr,
5031                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5032                                         &sample_action_pos, &qrss_action_pos);
5033         if (actions_n) {
5034                 /* The prefix actions must includes sample, tag, end. */
5035                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5036                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5037                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5038                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5039                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5040                                           item_size), 0, SOCKET_ID_ANY);
5041                 if (!sfx_actions)
5042                         return rte_flow_error_set(error, ENOMEM,
5043                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5044                                                   NULL, "no memory to split "
5045                                                   "sample flow");
5046                 /* The representor_id is -1 for uplink. */
5047                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5048                 if (!fdb_tx)
5049                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5050                                         + act_size);
5051                 pre_actions = sfx_actions + actions_n;
5052                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5053                                                 actions, sfx_actions,
5054                                                 pre_actions, actions_n,
5055                                                 sample_action_pos,
5056                                                 qrss_action_pos, error);
5057                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5058                         ret = -rte_errno;
5059                         goto exit;
5060                 }
5061                 /* Add the prefix subflow. */
5062                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5063                                               items, pre_actions,
5064                                               flow_split_info, error);
5065                 if (ret) {
5066                         ret = -rte_errno;
5067                         goto exit;
5068                 }
5069                 dev_flow->handle->split_flow_id = tag_id;
5070 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5071                 /* Set the sfx group attr. */
5072                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5073                                         dev_flow->dv.sample_res;
5074                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5075                                         sample_res->normal_path_tbl;
5076                 sfx_tbl_data = container_of(sfx_tbl,
5077                                         struct mlx5_flow_tbl_data_entry, tbl);
5078                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5079                 sfx_attr.group = sfx_attr.transfer ?
5080                                         (sfx_table_key.table_id - 1) :
5081                                          sfx_table_key.table_id;
5082                 flow_split_info->prefix_layers =
5083                                 flow_get_prefix_layer_flags(dev_flow);
5084                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5085                 /* Suffix group level already be scaled with factor, set
5086                  * skip_scale to 1 to avoid scale again in translation.
5087                  */
5088                 flow_split_info->skip_scale = 1;
5089 #endif
5090         }
5091         /* Add the suffix subflow. */
5092         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5093                                       sfx_items ? sfx_items : items,
5094                                       sfx_actions ? sfx_actions : actions,
5095                                       flow_split_info, error);
5096 exit:
5097         if (sfx_actions)
5098                 mlx5_free(sfx_actions);
5099         return ret;
5100 }
5101
5102 /**
5103  * Split the flow to subflow set. The splitters might be linked
5104  * in the chain, like this:
5105  * flow_create_split_outer() calls:
5106  *   flow_create_split_meter() calls:
5107  *     flow_create_split_metadata(meter_subflow_0) calls:
5108  *       flow_create_split_inner(metadata_subflow_0)
5109  *       flow_create_split_inner(metadata_subflow_1)
5110  *       flow_create_split_inner(metadata_subflow_2)
5111  *     flow_create_split_metadata(meter_subflow_1) calls:
5112  *       flow_create_split_inner(metadata_subflow_0)
5113  *       flow_create_split_inner(metadata_subflow_1)
5114  *       flow_create_split_inner(metadata_subflow_2)
5115  *
5116  * This provide flexible way to add new levels of flow splitting.
5117  * The all of successfully created subflows are included to the
5118  * parent flow dev_flow list.
5119  *
5120  * @param dev
5121  *   Pointer to Ethernet device.
5122  * @param[in] flow
5123  *   Parent flow structure pointer.
5124  * @param[in] attr
5125  *   Flow rule attributes.
5126  * @param[in] items
5127  *   Pattern specification (list terminated by the END pattern item).
5128  * @param[in] actions
5129  *   Associated actions (list terminated by the END action).
5130  * @param[in] flow_split_info
5131  *   Pointer to flow split info structure.
5132  * @param[out] error
5133  *   Perform verbose error reporting if not NULL.
5134  * @return
5135  *   0 on success, negative value otherwise
5136  */
5137 static int
5138 flow_create_split_outer(struct rte_eth_dev *dev,
5139                         struct rte_flow *flow,
5140                         const struct rte_flow_attr *attr,
5141                         const struct rte_flow_item items[],
5142                         const struct rte_flow_action actions[],
5143                         struct mlx5_flow_split_info *flow_split_info,
5144                         struct rte_flow_error *error)
5145 {
5146         int ret;
5147
5148         ret = flow_create_split_sample(dev, flow, attr, items,
5149                                        actions, flow_split_info, error);
5150         MLX5_ASSERT(ret <= 0);
5151         return ret;
5152 }
5153
5154 static struct mlx5_flow_tunnel *
5155 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5156                       const struct rte_flow_attr *attr,
5157                       const struct rte_flow_item items[],
5158                       const struct rte_flow_action actions[])
5159 {
5160         struct mlx5_flow_tunnel *tunnel;
5161
5162 #pragma GCC diagnostic push
5163 #pragma GCC diagnostic ignored "-Wcast-qual"
5164         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5165                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5166         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5167                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5168         else
5169                 tunnel = NULL;
5170 #pragma GCC diagnostic pop
5171
5172         return tunnel;
5173 }
5174
5175 /**
5176  * Adjust flow RSS workspace if needed.
5177  *
5178  * @param wks
5179  *   Pointer to thread flow work space.
5180  * @param rss_desc
5181  *   Pointer to RSS descriptor.
5182  * @param[in] nrssq_num
5183  *   New RSS queue number.
5184  *
5185  * @return
5186  *   0 on success, -1 otherwise and rte_errno is set.
5187  */
5188 static int
5189 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
5190                           struct mlx5_flow_rss_desc *rss_desc,
5191                           uint32_t nrssq_num)
5192 {
5193         if (likely(nrssq_num <= wks->rssq_num))
5194                 return 0;
5195         rss_desc->queue = realloc(rss_desc->queue,
5196                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
5197         if (!rss_desc->queue) {
5198                 rte_errno = ENOMEM;
5199                 return -1;
5200         }
5201         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
5202         return 0;
5203 }
5204
5205 /**
5206  * Create a flow and add it to @p list.
5207  *
5208  * @param dev
5209  *   Pointer to Ethernet device.
5210  * @param list
5211  *   Pointer to a TAILQ flow list. If this parameter NULL,
5212  *   no list insertion occurred, flow is just created,
5213  *   this is caller's responsibility to track the
5214  *   created flow.
5215  * @param[in] attr
5216  *   Flow rule attributes.
5217  * @param[in] items
5218  *   Pattern specification (list terminated by the END pattern item).
5219  * @param[in] actions
5220  *   Associated actions (list terminated by the END action).
5221  * @param[in] external
5222  *   This flow rule is created by request external to PMD.
5223  * @param[out] error
5224  *   Perform verbose error reporting if not NULL.
5225  *
5226  * @return
5227  *   A flow index on success, 0 otherwise and rte_errno is set.
5228  */
5229 static uint32_t
5230 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5231                  const struct rte_flow_attr *attr,
5232                  const struct rte_flow_item items[],
5233                  const struct rte_flow_action original_actions[],
5234                  bool external, struct rte_flow_error *error)
5235 {
5236         struct mlx5_priv *priv = dev->data->dev_private;
5237         struct rte_flow *flow = NULL;
5238         struct mlx5_flow *dev_flow;
5239         const struct rte_flow_action_rss *rss;
5240         struct mlx5_translated_shared_action
5241                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5242         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5243         union {
5244                 struct mlx5_flow_expand_rss buf;
5245                 uint8_t buffer[2048];
5246         } expand_buffer;
5247         union {
5248                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5249                 uint8_t buffer[2048];
5250         } actions_rx;
5251         union {
5252                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5253                 uint8_t buffer[2048];
5254         } actions_hairpin_tx;
5255         union {
5256                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5257                 uint8_t buffer[2048];
5258         } items_tx;
5259         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5260         struct mlx5_flow_rss_desc *rss_desc;
5261         const struct rte_flow_action *p_actions_rx;
5262         uint32_t i;
5263         uint32_t idx = 0;
5264         int hairpin_flow;
5265         struct rte_flow_attr attr_tx = { .priority = 0 };
5266         const struct rte_flow_action *actions;
5267         struct rte_flow_action *translated_actions = NULL;
5268         struct mlx5_flow_tunnel *tunnel;
5269         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5270         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
5271         struct mlx5_flow_split_info flow_split_info = {
5272                 .external = !!external,
5273                 .skip_scale = 0,
5274                 .flow_idx = 0,
5275                 .prefix_mark = 0,
5276                 .prefix_layers = 0
5277         };
5278         int ret;
5279
5280         MLX5_ASSERT(wks);
5281         rss_desc = &wks->rss_desc;
5282         ret = flow_shared_actions_translate(dev, original_actions,
5283                                             shared_actions,
5284                                             &shared_actions_n,
5285                                             &translated_actions, error);
5286         if (ret < 0) {
5287                 MLX5_ASSERT(translated_actions == NULL);
5288                 return 0;
5289         }
5290         actions = translated_actions ? translated_actions : original_actions;
5291         p_actions_rx = actions;
5292         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5293         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
5294                                 external, hairpin_flow, error);
5295         if (ret < 0)
5296                 goto error_before_hairpin_split;
5297         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5298         if (!flow) {
5299                 rte_errno = ENOMEM;
5300                 goto error_before_hairpin_split;
5301         }
5302         if (hairpin_flow > 0) {
5303                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5304                         rte_errno = EINVAL;
5305                         goto error_before_hairpin_split;
5306                 }
5307                 flow_hairpin_split(dev, actions, actions_rx.actions,
5308                                    actions_hairpin_tx.actions, items_tx.items,
5309                                    idx);
5310                 p_actions_rx = actions_rx.actions;
5311         }
5312         flow_split_info.flow_idx = idx;
5313         flow->drv_type = flow_get_drv_type(dev, attr);
5314         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5315                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5316         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
5317         rss = flow_get_rss_action(p_actions_rx);
5318         if (rss) {
5319                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
5320                         return 0;
5321                 /*
5322                  * The following information is required by
5323                  * mlx5_flow_hashfields_adjust() in advance.
5324                  */
5325                 rss_desc->level = rss->level;
5326                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5327                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5328         }
5329         flow->dev_handles = 0;
5330         if (rss && rss->types) {
5331                 unsigned int graph_root;
5332
5333                 graph_root = find_graph_root(items, rss->level);
5334                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5335                                            items, rss->types,
5336                                            mlx5_support_expansion, graph_root);
5337                 MLX5_ASSERT(ret > 0 &&
5338                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5339         } else {
5340                 buf->entries = 1;
5341                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5342         }
5343         rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions,
5344                                                       shared_actions_n);
5345         for (i = 0; i < buf->entries; ++i) {
5346                 /* Initialize flow split data. */
5347                 flow_split_info.prefix_layers = 0;
5348                 flow_split_info.prefix_mark = 0;
5349                 flow_split_info.skip_scale = 0;
5350                 /*
5351                  * The splitter may create multiple dev_flows,
5352                  * depending on configuration. In the simplest
5353                  * case it just creates unmodified original flow.
5354                  */
5355                 ret = flow_create_split_outer(dev, flow, attr,
5356                                               buf->entry[i].pattern,
5357                                               p_actions_rx, &flow_split_info,
5358                                               error);
5359                 if (ret < 0)
5360                         goto error;
5361                 if (is_flow_tunnel_steer_rule(dev, attr,
5362                                               buf->entry[i].pattern,
5363                                               p_actions_rx)) {
5364                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5365                                                            p_actions_rx,
5366                                                            idx,
5367                                                            &default_miss_ctx,
5368                                                            error);
5369                         if (ret < 0) {
5370                                 mlx5_free(default_miss_ctx.queue);
5371                                 goto error;
5372                         }
5373                 }
5374         }
5375         /* Create the tx flow. */
5376         if (hairpin_flow) {
5377                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5378                 attr_tx.ingress = 0;
5379                 attr_tx.egress = 1;
5380                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5381                                          actions_hairpin_tx.actions,
5382                                          idx, error);
5383                 if (!dev_flow)
5384                         goto error;
5385                 dev_flow->flow = flow;
5386                 dev_flow->external = 0;
5387                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5388                               dev_flow->handle, next);
5389                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5390                                          items_tx.items,
5391                                          actions_hairpin_tx.actions, error);
5392                 if (ret < 0)
5393                         goto error;
5394         }
5395         /*
5396          * Update the metadata register copy table. If extensive
5397          * metadata feature is enabled and registers are supported
5398          * we might create the extra rte_flow for each unique
5399          * MARK/FLAG action ID.
5400          *
5401          * The table is updated for ingress Flows only, because
5402          * the egress Flows belong to the different device and
5403          * copy table should be updated in peer NIC Rx domain.
5404          */
5405         if (attr->ingress &&
5406             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5407                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5408                 if (ret)
5409                         goto error;
5410         }
5411         /*
5412          * If the flow is external (from application) OR device is started,
5413          * OR mreg discover, then apply immediately.
5414          */
5415         if (external || dev->data->dev_started ||
5416             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
5417              attr->priority == MLX5_FLOW_PRIO_RSVD)) {
5418                 ret = flow_drv_apply(dev, flow, error);
5419                 if (ret < 0)
5420                         goto error;
5421         }
5422         if (list) {
5423                 rte_spinlock_lock(&priv->flow_list_lock);
5424                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5425                              flow, next);
5426                 rte_spinlock_unlock(&priv->flow_list_lock);
5427         }
5428         flow_rxq_flags_set(dev, flow);
5429         rte_free(translated_actions);
5430         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5431         if (tunnel) {
5432                 flow->tunnel = 1;
5433                 flow->tunnel_id = tunnel->tunnel_id;
5434                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5435                 mlx5_free(default_miss_ctx.queue);
5436         }
5437         mlx5_flow_pop_thread_workspace();
5438         return idx;
5439 error:
5440         MLX5_ASSERT(flow);
5441         ret = rte_errno; /* Save rte_errno before cleanup. */
5442         flow_mreg_del_copy_action(dev, flow);
5443         flow_drv_destroy(dev, flow);
5444         if (rss_desc->shared_rss)
5445                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
5446                         mlx5_ipool_get
5447                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
5448                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
5449         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5450         rte_errno = ret; /* Restore rte_errno. */
5451         ret = rte_errno;
5452         rte_errno = ret;
5453         mlx5_flow_pop_thread_workspace();
5454 error_before_hairpin_split:
5455         rte_free(translated_actions);
5456         return 0;
5457 }
5458
5459 /**
5460  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5461  * incoming packets to table 1.
5462  *
5463  * Other flow rules, requested for group n, will be created in
5464  * e-switch table n+1.
5465  * Jump action to e-switch group n will be created to group n+1.
5466  *
5467  * Used when working in switchdev mode, to utilise advantages of table 1
5468  * and above.
5469  *
5470  * @param dev
5471  *   Pointer to Ethernet device.
5472  *
5473  * @return
5474  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5475  */
5476 struct rte_flow *
5477 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5478 {
5479         const struct rte_flow_attr attr = {
5480                 .group = 0,
5481                 .priority = 0,
5482                 .ingress = 1,
5483                 .egress = 0,
5484                 .transfer = 1,
5485         };
5486         const struct rte_flow_item pattern = {
5487                 .type = RTE_FLOW_ITEM_TYPE_END,
5488         };
5489         struct rte_flow_action_jump jump = {
5490                 .group = 1,
5491         };
5492         const struct rte_flow_action actions[] = {
5493                 {
5494                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5495                         .conf = &jump,
5496                 },
5497                 {
5498                         .type = RTE_FLOW_ACTION_TYPE_END,
5499                 },
5500         };
5501         struct mlx5_priv *priv = dev->data->dev_private;
5502         struct rte_flow_error error;
5503
5504         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5505                                                    &attr, &pattern,
5506                                                    actions, false, &error);
5507 }
5508
5509 /**
5510  * Validate a flow supported by the NIC.
5511  *
5512  * @see rte_flow_validate()
5513  * @see rte_flow_ops
5514  */
5515 int
5516 mlx5_flow_validate(struct rte_eth_dev *dev,
5517                    const struct rte_flow_attr *attr,
5518                    const struct rte_flow_item items[],
5519                    const struct rte_flow_action original_actions[],
5520                    struct rte_flow_error *error)
5521 {
5522         int hairpin_flow;
5523         struct mlx5_translated_shared_action
5524                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5525         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5526         const struct rte_flow_action *actions;
5527         struct rte_flow_action *translated_actions = NULL;
5528         int ret = flow_shared_actions_translate(dev, original_actions,
5529                                                 shared_actions,
5530                                                 &shared_actions_n,
5531                                                 &translated_actions, error);
5532
5533         if (ret)
5534                 return ret;
5535         actions = translated_actions ? translated_actions : original_actions;
5536         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5537         ret = flow_drv_validate(dev, attr, items, actions,
5538                                 true, hairpin_flow, error);
5539         rte_free(translated_actions);
5540         return ret;
5541 }
5542
5543 /**
5544  * Create a flow.
5545  *
5546  * @see rte_flow_create()
5547  * @see rte_flow_ops
5548  */
5549 struct rte_flow *
5550 mlx5_flow_create(struct rte_eth_dev *dev,
5551                  const struct rte_flow_attr *attr,
5552                  const struct rte_flow_item items[],
5553                  const struct rte_flow_action actions[],
5554                  struct rte_flow_error *error)
5555 {
5556         struct mlx5_priv *priv = dev->data->dev_private;
5557
5558         /*
5559          * If the device is not started yet, it is not allowed to created a
5560          * flow from application. PMD default flows and traffic control flows
5561          * are not affected.
5562          */
5563         if (unlikely(!dev->data->dev_started)) {
5564                 DRV_LOG(DEBUG, "port %u is not started when "
5565                         "inserting a flow", dev->data->port_id);
5566                 rte_flow_error_set(error, ENODEV,
5567                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5568                                    NULL,
5569                                    "port not started");
5570                 return NULL;
5571         }
5572
5573         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5574                                   attr, items, actions, true, error);
5575 }
5576
5577 /**
5578  * Destroy a flow in a list.
5579  *
5580  * @param dev
5581  *   Pointer to Ethernet device.
5582  * @param list
5583  *   Pointer to the Indexed flow list. If this parameter NULL,
5584  *   there is no flow removal from the list. Be noted that as
5585  *   flow is add to the indexed list, memory of the indexed
5586  *   list points to maybe changed as flow destroyed.
5587  * @param[in] flow_idx
5588  *   Index of flow to destroy.
5589  */
5590 static void
5591 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5592                   uint32_t flow_idx)
5593 {
5594         struct mlx5_priv *priv = dev->data->dev_private;
5595         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5596                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5597
5598         if (!flow)
5599                 return;
5600         /*
5601          * Update RX queue flags only if port is started, otherwise it is
5602          * already clean.
5603          */
5604         if (dev->data->dev_started)
5605                 flow_rxq_flags_trim(dev, flow);
5606         flow_drv_destroy(dev, flow);
5607         if (list) {
5608                 rte_spinlock_lock(&priv->flow_list_lock);
5609                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5610                              flow_idx, flow, next);
5611                 rte_spinlock_unlock(&priv->flow_list_lock);
5612         }
5613         if (flow->tunnel) {
5614                 struct mlx5_flow_tunnel *tunnel;
5615
5616                 rte_spinlock_lock(&mlx5_tunnel_hub(dev)->sl);
5617                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5618                 RTE_VERIFY(tunnel);
5619                 LIST_REMOVE(tunnel, chain);
5620                 rte_spinlock_unlock(&mlx5_tunnel_hub(dev)->sl);
5621                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5622                         mlx5_flow_tunnel_free(dev, tunnel);
5623         }
5624         flow_mreg_del_copy_action(dev, flow);
5625         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5626 }
5627
5628 /**
5629  * Destroy all flows.
5630  *
5631  * @param dev
5632  *   Pointer to Ethernet device.
5633  * @param list
5634  *   Pointer to the Indexed flow list.
5635  * @param active
5636  *   If flushing is called avtively.
5637  */
5638 void
5639 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5640 {
5641         uint32_t num_flushed = 0;
5642
5643         while (*list) {
5644                 flow_list_destroy(dev, list, *list);
5645                 num_flushed++;
5646         }
5647         if (active) {
5648                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5649                         dev->data->port_id, num_flushed);
5650         }
5651 }
5652
5653 /**
5654  * Stop all default actions for flows.
5655  *
5656  * @param dev
5657  *   Pointer to Ethernet device.
5658  */
5659 void
5660 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5661 {
5662         flow_mreg_del_default_copy_action(dev);
5663         flow_rxq_flags_clear(dev);
5664 }
5665
5666 /**
5667  * Start all default actions for flows.
5668  *
5669  * @param dev
5670  *   Pointer to Ethernet device.
5671  * @return
5672  *   0 on success, a negative errno value otherwise and rte_errno is set.
5673  */
5674 int
5675 mlx5_flow_start_default(struct rte_eth_dev *dev)
5676 {
5677         struct rte_flow_error error;
5678
5679         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5680         return flow_mreg_add_default_copy_action(dev, &error);
5681 }
5682
5683 /**
5684  * Release key of thread specific flow workspace data.
5685  */
5686 static void
5687 flow_release_workspace(void *data)
5688 {
5689         struct mlx5_flow_workspace *wks = data;
5690         struct mlx5_flow_workspace *next;
5691
5692         while (wks) {
5693                 next = wks->next;
5694                 free(wks->rss_desc.queue);
5695                 free(wks);
5696                 wks = next;
5697         }
5698 }
5699
5700 /**
5701  * Initialize key of thread specific flow workspace data.
5702  */
5703 static void
5704 flow_alloc_workspace(void)
5705 {
5706         if (pthread_key_create(&key_workspace, flow_release_workspace))
5707                 DRV_LOG(ERR, "Can't create flow workspace data thread key.");
5708 }
5709
5710 /**
5711  * Get thread specific current flow workspace.
5712  *
5713  * @return pointer to thread specific flow workspace data, NULL on error.
5714  */
5715 struct mlx5_flow_workspace*
5716 mlx5_flow_get_thread_workspace(void)
5717 {
5718         struct mlx5_flow_workspace *data;
5719
5720         data = pthread_getspecific(key_workspace);
5721         MLX5_ASSERT(data && data->inuse);
5722         if (!data || !data->inuse)
5723                 DRV_LOG(ERR, "flow workspace not initialized.");
5724         return data;
5725 }
5726
5727 /**
5728  * Allocate and init new flow workspace.
5729  *
5730  * @return pointer to flow workspace data, NULL on error.
5731  */
5732 static struct mlx5_flow_workspace*
5733 flow_alloc_thread_workspace(void)
5734 {
5735         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
5736
5737         if (!data) {
5738                 DRV_LOG(ERR, "Failed to allocate flow workspace "
5739                         "memory.");
5740                 return NULL;
5741         }
5742         data->rss_desc.queue = calloc(1,
5743                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
5744         if (!data->rss_desc.queue)
5745                 goto err;
5746         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
5747         return data;
5748 err:
5749         if (data->rss_desc.queue)
5750                 free(data->rss_desc.queue);
5751         free(data);
5752         return NULL;
5753 }
5754
5755 /**
5756  * Get new thread specific flow workspace.
5757  *
5758  * If current workspace inuse, create new one and set as current.
5759  *
5760  * @return pointer to thread specific flow workspace data, NULL on error.
5761  */
5762 static struct mlx5_flow_workspace*
5763 mlx5_flow_push_thread_workspace(void)
5764 {
5765         struct mlx5_flow_workspace *curr;
5766         struct mlx5_flow_workspace *data;
5767
5768         if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
5769                 DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
5770                 return NULL;
5771         }
5772         curr = pthread_getspecific(key_workspace);
5773         if (!curr) {
5774                 data = flow_alloc_thread_workspace();
5775                 if (!data)
5776                         return NULL;
5777         } else if (!curr->inuse) {
5778                 data = curr;
5779         } else if (curr->next) {
5780                 data = curr->next;
5781         } else {
5782                 data = flow_alloc_thread_workspace();
5783                 if (!data)
5784                         return NULL;
5785                 curr->next = data;
5786                 data->prev = curr;
5787         }
5788         data->inuse = 1;
5789         data->flow_idx = 0;
5790         /* Set as current workspace */
5791         if (pthread_setspecific(key_workspace, data))
5792                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5793         return data;
5794 }
5795
5796 /**
5797  * Close current thread specific flow workspace.
5798  *
5799  * If previous workspace available, set it as current.
5800  *
5801  * @return pointer to thread specific flow workspace data, NULL on error.
5802  */
5803 static void
5804 mlx5_flow_pop_thread_workspace(void)
5805 {
5806         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
5807
5808         if (!data)
5809                 return;
5810         if (!data->inuse) {
5811                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
5812                 return;
5813         }
5814         data->inuse = 0;
5815         if (!data->prev)
5816                 return;
5817         if (pthread_setspecific(key_workspace, data->prev))
5818                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5819 }
5820
5821 /**
5822  * Verify the flow list is empty
5823  *
5824  * @param dev
5825  *  Pointer to Ethernet device.
5826  *
5827  * @return the number of flows not released.
5828  */
5829 int
5830 mlx5_flow_verify(struct rte_eth_dev *dev)
5831 {
5832         struct mlx5_priv *priv = dev->data->dev_private;
5833         struct rte_flow *flow;
5834         uint32_t idx;
5835         int ret = 0;
5836
5837         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5838                       flow, next) {
5839                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5840                         dev->data->port_id, (void *)flow);
5841                 ++ret;
5842         }
5843         return ret;
5844 }
5845
5846 /**
5847  * Enable default hairpin egress flow.
5848  *
5849  * @param dev
5850  *   Pointer to Ethernet device.
5851  * @param queue
5852  *   The queue index.
5853  *
5854  * @return
5855  *   0 on success, a negative errno value otherwise and rte_errno is set.
5856  */
5857 int
5858 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5859                             uint32_t queue)
5860 {
5861         struct mlx5_priv *priv = dev->data->dev_private;
5862         const struct rte_flow_attr attr = {
5863                 .egress = 1,
5864                 .priority = 0,
5865         };
5866         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5867                 .queue = queue,
5868         };
5869         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5870                 .queue = UINT32_MAX,
5871         };
5872         struct rte_flow_item items[] = {
5873                 {
5874                         .type = (enum rte_flow_item_type)
5875                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5876                         .spec = &queue_spec,
5877                         .last = NULL,
5878                         .mask = &queue_mask,
5879                 },
5880                 {
5881                         .type = RTE_FLOW_ITEM_TYPE_END,
5882                 },
5883         };
5884         struct rte_flow_action_jump jump = {
5885                 .group = MLX5_HAIRPIN_TX_TABLE,
5886         };
5887         struct rte_flow_action actions[2];
5888         uint32_t flow_idx;
5889         struct rte_flow_error error;
5890
5891         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5892         actions[0].conf = &jump;
5893         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5894         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5895                                 &attr, items, actions, false, &error);
5896         if (!flow_idx) {
5897                 DRV_LOG(DEBUG,
5898                         "Failed to create ctrl flow: rte_errno(%d),"
5899                         " type(%d), message(%s)",
5900                         rte_errno, error.type,
5901                         error.message ? error.message : " (no stated reason)");
5902                 return -rte_errno;
5903         }
5904         return 0;
5905 }
5906
5907 /**
5908  * Enable a control flow configured from the control plane.
5909  *
5910  * @param dev
5911  *   Pointer to Ethernet device.
5912  * @param eth_spec
5913  *   An Ethernet flow spec to apply.
5914  * @param eth_mask
5915  *   An Ethernet flow mask to apply.
5916  * @param vlan_spec
5917  *   A VLAN flow spec to apply.
5918  * @param vlan_mask
5919  *   A VLAN flow mask to apply.
5920  *
5921  * @return
5922  *   0 on success, a negative errno value otherwise and rte_errno is set.
5923  */
5924 int
5925 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5926                     struct rte_flow_item_eth *eth_spec,
5927                     struct rte_flow_item_eth *eth_mask,
5928                     struct rte_flow_item_vlan *vlan_spec,
5929                     struct rte_flow_item_vlan *vlan_mask)
5930 {
5931         struct mlx5_priv *priv = dev->data->dev_private;
5932         const struct rte_flow_attr attr = {
5933                 .ingress = 1,
5934                 .priority = MLX5_FLOW_PRIO_RSVD,
5935         };
5936         struct rte_flow_item items[] = {
5937                 {
5938                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5939                         .spec = eth_spec,
5940                         .last = NULL,
5941                         .mask = eth_mask,
5942                 },
5943                 {
5944                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5945                                               RTE_FLOW_ITEM_TYPE_END,
5946                         .spec = vlan_spec,
5947                         .last = NULL,
5948                         .mask = vlan_mask,
5949                 },
5950                 {
5951                         .type = RTE_FLOW_ITEM_TYPE_END,
5952                 },
5953         };
5954         uint16_t queue[priv->reta_idx_n];
5955         struct rte_flow_action_rss action_rss = {
5956                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5957                 .level = 0,
5958                 .types = priv->rss_conf.rss_hf,
5959                 .key_len = priv->rss_conf.rss_key_len,
5960                 .queue_num = priv->reta_idx_n,
5961                 .key = priv->rss_conf.rss_key,
5962                 .queue = queue,
5963         };
5964         struct rte_flow_action actions[] = {
5965                 {
5966                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5967                         .conf = &action_rss,
5968                 },
5969                 {
5970                         .type = RTE_FLOW_ACTION_TYPE_END,
5971                 },
5972         };
5973         uint32_t flow_idx;
5974         struct rte_flow_error error;
5975         unsigned int i;
5976
5977         if (!priv->reta_idx_n || !priv->rxqs_n) {
5978                 return 0;
5979         }
5980         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5981                 action_rss.types = 0;
5982         for (i = 0; i != priv->reta_idx_n; ++i)
5983                 queue[i] = (*priv->reta_idx)[i];
5984         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5985                                 &attr, items, actions, false, &error);
5986         if (!flow_idx)
5987                 return -rte_errno;
5988         return 0;
5989 }
5990
5991 /**
5992  * Enable a flow control configured from the control plane.
5993  *
5994  * @param dev
5995  *   Pointer to Ethernet device.
5996  * @param eth_spec
5997  *   An Ethernet flow spec to apply.
5998  * @param eth_mask
5999  *   An Ethernet flow mask to apply.
6000  *
6001  * @return
6002  *   0 on success, a negative errno value otherwise and rte_errno is set.
6003  */
6004 int
6005 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6006                struct rte_flow_item_eth *eth_spec,
6007                struct rte_flow_item_eth *eth_mask)
6008 {
6009         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6010 }
6011
6012 /**
6013  * Create default miss flow rule matching lacp traffic
6014  *
6015  * @param dev
6016  *   Pointer to Ethernet device.
6017  * @param eth_spec
6018  *   An Ethernet flow spec to apply.
6019  *
6020  * @return
6021  *   0 on success, a negative errno value otherwise and rte_errno is set.
6022  */
6023 int
6024 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6025 {
6026         struct mlx5_priv *priv = dev->data->dev_private;
6027         /*
6028          * The LACP matching is done by only using ether type since using
6029          * a multicast dst mac causes kernel to give low priority to this flow.
6030          */
6031         static const struct rte_flow_item_eth lacp_spec = {
6032                 .type = RTE_BE16(0x8809),
6033         };
6034         static const struct rte_flow_item_eth lacp_mask = {
6035                 .type = 0xffff,
6036         };
6037         const struct rte_flow_attr attr = {
6038                 .ingress = 1,
6039         };
6040         struct rte_flow_item items[] = {
6041                 {
6042                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6043                         .spec = &lacp_spec,
6044                         .mask = &lacp_mask,
6045                 },
6046                 {
6047                         .type = RTE_FLOW_ITEM_TYPE_END,
6048                 },
6049         };
6050         struct rte_flow_action actions[] = {
6051                 {
6052                         .type = (enum rte_flow_action_type)
6053                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6054                 },
6055                 {
6056                         .type = RTE_FLOW_ACTION_TYPE_END,
6057                 },
6058         };
6059         struct rte_flow_error error;
6060         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6061                                 &attr, items, actions, false, &error);
6062
6063         if (!flow_idx)
6064                 return -rte_errno;
6065         return 0;
6066 }
6067
6068 /**
6069  * Destroy a flow.
6070  *
6071  * @see rte_flow_destroy()
6072  * @see rte_flow_ops
6073  */
6074 int
6075 mlx5_flow_destroy(struct rte_eth_dev *dev,
6076                   struct rte_flow *flow,
6077                   struct rte_flow_error *error __rte_unused)
6078 {
6079         struct mlx5_priv *priv = dev->data->dev_private;
6080
6081         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6082         return 0;
6083 }
6084
6085 /**
6086  * Destroy all flows.
6087  *
6088  * @see rte_flow_flush()
6089  * @see rte_flow_ops
6090  */
6091 int
6092 mlx5_flow_flush(struct rte_eth_dev *dev,
6093                 struct rte_flow_error *error __rte_unused)
6094 {
6095         struct mlx5_priv *priv = dev->data->dev_private;
6096
6097         mlx5_flow_list_flush(dev, &priv->flows, false);
6098         return 0;
6099 }
6100
6101 /**
6102  * Isolated mode.
6103  *
6104  * @see rte_flow_isolate()
6105  * @see rte_flow_ops
6106  */
6107 int
6108 mlx5_flow_isolate(struct rte_eth_dev *dev,
6109                   int enable,
6110                   struct rte_flow_error *error)
6111 {
6112         struct mlx5_priv *priv = dev->data->dev_private;
6113
6114         if (dev->data->dev_started) {
6115                 rte_flow_error_set(error, EBUSY,
6116                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6117                                    NULL,
6118                                    "port must be stopped first");
6119                 return -rte_errno;
6120         }
6121         priv->isolated = !!enable;
6122         if (enable)
6123                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6124         else
6125                 dev->dev_ops = &mlx5_os_dev_ops;
6126
6127         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6128         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6129
6130         return 0;
6131 }
6132
6133 /**
6134  * Query a flow.
6135  *
6136  * @see rte_flow_query()
6137  * @see rte_flow_ops
6138  */
6139 static int
6140 flow_drv_query(struct rte_eth_dev *dev,
6141                uint32_t flow_idx,
6142                const struct rte_flow_action *actions,
6143                void *data,
6144                struct rte_flow_error *error)
6145 {
6146         struct mlx5_priv *priv = dev->data->dev_private;
6147         const struct mlx5_flow_driver_ops *fops;
6148         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6149                                                [MLX5_IPOOL_RTE_FLOW],
6150                                                flow_idx);
6151         enum mlx5_flow_drv_type ftype;
6152
6153         if (!flow) {
6154                 return rte_flow_error_set(error, ENOENT,
6155                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6156                           NULL,
6157                           "invalid flow handle");
6158         }
6159         ftype = flow->drv_type;
6160         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6161         fops = flow_get_drv_ops(ftype);
6162
6163         return fops->query(dev, flow, actions, data, error);
6164 }
6165
6166 /**
6167  * Query a flow.
6168  *
6169  * @see rte_flow_query()
6170  * @see rte_flow_ops
6171  */
6172 int
6173 mlx5_flow_query(struct rte_eth_dev *dev,
6174                 struct rte_flow *flow,
6175                 const struct rte_flow_action *actions,
6176                 void *data,
6177                 struct rte_flow_error *error)
6178 {
6179         int ret;
6180
6181         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6182                              error);
6183         if (ret < 0)
6184                 return ret;
6185         return 0;
6186 }
6187
6188 /**
6189  * Manage filter operations.
6190  *
6191  * @param dev
6192  *   Pointer to Ethernet device structure.
6193  * @param filter_type
6194  *   Filter type.
6195  * @param filter_op
6196  *   Operation to perform.
6197  * @param arg
6198  *   Pointer to operation-specific structure.
6199  *
6200  * @return
6201  *   0 on success, a negative errno value otherwise and rte_errno is set.
6202  */
6203 int
6204 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6205                      enum rte_filter_type filter_type,
6206                      enum rte_filter_op filter_op,
6207                      void *arg)
6208 {
6209         switch (filter_type) {
6210         case RTE_ETH_FILTER_GENERIC:
6211                 if (filter_op != RTE_ETH_FILTER_GET) {
6212                         rte_errno = EINVAL;
6213                         return -rte_errno;
6214                 }
6215                 *(const void **)arg = &mlx5_flow_ops;
6216                 return 0;
6217         default:
6218                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6219                         dev->data->port_id, filter_type);
6220                 rte_errno = ENOTSUP;
6221                 return -rte_errno;
6222         }
6223         return 0;
6224 }
6225
6226 /**
6227  * Create the needed meter and suffix tables.
6228  *
6229  * @param[in] dev
6230  *   Pointer to Ethernet device.
6231  * @param[in] fm
6232  *   Pointer to the flow meter.
6233  *
6234  * @return
6235  *   Pointer to table set on success, NULL otherwise.
6236  */
6237 struct mlx5_meter_domains_infos *
6238 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6239                           const struct mlx5_flow_meter *fm)
6240 {
6241         const struct mlx5_flow_driver_ops *fops;
6242
6243         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6244         return fops->create_mtr_tbls(dev, fm);
6245 }
6246
6247 /**
6248  * Destroy the meter table set.
6249  *
6250  * @param[in] dev
6251  *   Pointer to Ethernet device.
6252  * @param[in] tbl
6253  *   Pointer to the meter table set.
6254  *
6255  * @return
6256  *   0 on success.
6257  */
6258 int
6259 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6260                            struct mlx5_meter_domains_infos *tbls)
6261 {
6262         const struct mlx5_flow_driver_ops *fops;
6263
6264         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6265         return fops->destroy_mtr_tbls(dev, tbls);
6266 }
6267
6268 /**
6269  * Create policer rules.
6270  *
6271  * @param[in] dev
6272  *   Pointer to Ethernet device.
6273  * @param[in] fm
6274  *   Pointer to flow meter structure.
6275  * @param[in] attr
6276  *   Pointer to flow attributes.
6277  *
6278  * @return
6279  *   0 on success, -1 otherwise.
6280  */
6281 int
6282 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6283                                struct mlx5_flow_meter *fm,
6284                                const struct rte_flow_attr *attr)
6285 {
6286         const struct mlx5_flow_driver_ops *fops;
6287
6288         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6289         return fops->create_policer_rules(dev, fm, attr);
6290 }
6291
6292 /**
6293  * Destroy policer rules.
6294  *
6295  * @param[in] fm
6296  *   Pointer to flow meter structure.
6297  * @param[in] attr
6298  *   Pointer to flow attributes.
6299  *
6300  * @return
6301  *   0 on success, -1 otherwise.
6302  */
6303 int
6304 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6305                                 struct mlx5_flow_meter *fm,
6306                                 const struct rte_flow_attr *attr)
6307 {
6308         const struct mlx5_flow_driver_ops *fops;
6309
6310         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6311         return fops->destroy_policer_rules(dev, fm, attr);
6312 }
6313
6314 /**
6315  * Allocate a counter.
6316  *
6317  * @param[in] dev
6318  *   Pointer to Ethernet device structure.
6319  *
6320  * @return
6321  *   Index to allocated counter  on success, 0 otherwise.
6322  */
6323 uint32_t
6324 mlx5_counter_alloc(struct rte_eth_dev *dev)
6325 {
6326         const struct mlx5_flow_driver_ops *fops;
6327         struct rte_flow_attr attr = { .transfer = 0 };
6328
6329         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6330                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6331                 return fops->counter_alloc(dev);
6332         }
6333         DRV_LOG(ERR,
6334                 "port %u counter allocate is not supported.",
6335                  dev->data->port_id);
6336         return 0;
6337 }
6338
6339 /**
6340  * Free a counter.
6341  *
6342  * @param[in] dev
6343  *   Pointer to Ethernet device structure.
6344  * @param[in] cnt
6345  *   Index to counter to be free.
6346  */
6347 void
6348 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6349 {
6350         const struct mlx5_flow_driver_ops *fops;
6351         struct rte_flow_attr attr = { .transfer = 0 };
6352
6353         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6354                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6355                 fops->counter_free(dev, cnt);
6356                 return;
6357         }
6358         DRV_LOG(ERR,
6359                 "port %u counter free is not supported.",
6360                  dev->data->port_id);
6361 }
6362
6363 /**
6364  * Query counter statistics.
6365  *
6366  * @param[in] dev
6367  *   Pointer to Ethernet device structure.
6368  * @param[in] cnt
6369  *   Index to counter to query.
6370  * @param[in] clear
6371  *   Set to clear counter statistics.
6372  * @param[out] pkts
6373  *   The counter hits packets number to save.
6374  * @param[out] bytes
6375  *   The counter hits bytes number to save.
6376  *
6377  * @return
6378  *   0 on success, a negative errno value otherwise.
6379  */
6380 int
6381 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6382                    bool clear, uint64_t *pkts, uint64_t *bytes)
6383 {
6384         const struct mlx5_flow_driver_ops *fops;
6385         struct rte_flow_attr attr = { .transfer = 0 };
6386
6387         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6388                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6389                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6390         }
6391         DRV_LOG(ERR,
6392                 "port %u counter query is not supported.",
6393                  dev->data->port_id);
6394         return -ENOTSUP;
6395 }
6396
6397 /**
6398  * Allocate a new memory for the counter values wrapped by all the needed
6399  * management.
6400  *
6401  * @param[in] sh
6402  *   Pointer to mlx5_dev_ctx_shared object.
6403  *
6404  * @return
6405  *   0 on success, a negative errno value otherwise.
6406  */
6407 static int
6408 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
6409 {
6410         struct mlx5_devx_mkey_attr mkey_attr;
6411         struct mlx5_counter_stats_mem_mng *mem_mng;
6412         volatile struct flow_counter_stats *raw_data;
6413         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
6414         int size = (sizeof(struct flow_counter_stats) *
6415                         MLX5_COUNTERS_PER_POOL +
6416                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
6417                         sizeof(struct mlx5_counter_stats_mem_mng);
6418         size_t pgsize = rte_mem_page_size();
6419         uint8_t *mem;
6420         int i;
6421
6422         if (pgsize == (size_t)-1) {
6423                 DRV_LOG(ERR, "Failed to get mem page size");
6424                 rte_errno = ENOMEM;
6425                 return -ENOMEM;
6426         }
6427         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
6428         if (!mem) {
6429                 rte_errno = ENOMEM;
6430                 return -ENOMEM;
6431         }
6432         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
6433         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
6434         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
6435                                                  IBV_ACCESS_LOCAL_WRITE);
6436         if (!mem_mng->umem) {
6437                 rte_errno = errno;
6438                 mlx5_free(mem);
6439                 return -rte_errno;
6440         }
6441         mkey_attr.addr = (uintptr_t)mem;
6442         mkey_attr.size = size;
6443         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
6444         mkey_attr.pd = sh->pdn;
6445         mkey_attr.log_entity_size = 0;
6446         mkey_attr.pg_access = 0;
6447         mkey_attr.klm_array = NULL;
6448         mkey_attr.klm_num = 0;
6449         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
6450         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
6451         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
6452         if (!mem_mng->dm) {
6453                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
6454                 rte_errno = errno;
6455                 mlx5_free(mem);
6456                 return -rte_errno;
6457         }
6458         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
6459         raw_data = (volatile struct flow_counter_stats *)mem;
6460         for (i = 0; i < raws_n; ++i) {
6461                 mem_mng->raws[i].mem_mng = mem_mng;
6462                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
6463         }
6464         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
6465                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
6466                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
6467                                  next);
6468         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
6469         sh->cmng.mem_mng = mem_mng;
6470         return 0;
6471 }
6472
6473 /**
6474  * Set the statistic memory to the new counter pool.
6475  *
6476  * @param[in] sh
6477  *   Pointer to mlx5_dev_ctx_shared object.
6478  * @param[in] pool
6479  *   Pointer to the pool to set the statistic memory.
6480  *
6481  * @return
6482  *   0 on success, a negative errno value otherwise.
6483  */
6484 static int
6485 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
6486                                struct mlx5_flow_counter_pool *pool)
6487 {
6488         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6489         /* Resize statistic memory once used out. */
6490         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
6491             mlx5_flow_create_counter_stat_mem_mng(sh)) {
6492                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
6493                 return -1;
6494         }
6495         rte_spinlock_lock(&pool->sl);
6496         pool->raw = cmng->mem_mng->raws + pool->index %
6497                     MLX5_CNT_CONTAINER_RESIZE;
6498         rte_spinlock_unlock(&pool->sl);
6499         pool->raw_hw = NULL;
6500         return 0;
6501 }
6502
6503 #define MLX5_POOL_QUERY_FREQ_US 1000000
6504
6505 /**
6506  * Set the periodic procedure for triggering asynchronous batch queries for all
6507  * the counter pools.
6508  *
6509  * @param[in] sh
6510  *   Pointer to mlx5_dev_ctx_shared object.
6511  */
6512 void
6513 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6514 {
6515         uint32_t pools_n, us;
6516
6517         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
6518         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6519         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6520         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6521                 sh->cmng.query_thread_on = 0;
6522                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6523         } else {
6524                 sh->cmng.query_thread_on = 1;
6525         }
6526 }
6527
6528 /**
6529  * The periodic procedure for triggering asynchronous batch queries for all the
6530  * counter pools. This function is probably called by the host thread.
6531  *
6532  * @param[in] arg
6533  *   The parameter for the alarm process.
6534  */
6535 void
6536 mlx5_flow_query_alarm(void *arg)
6537 {
6538         struct mlx5_dev_ctx_shared *sh = arg;
6539         int ret;
6540         uint16_t pool_index = sh->cmng.pool_index;
6541         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6542         struct mlx5_flow_counter_pool *pool;
6543         uint16_t n_valid;
6544
6545         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6546                 goto set_alarm;
6547         rte_spinlock_lock(&cmng->pool_update_sl);
6548         pool = cmng->pools[pool_index];
6549         n_valid = cmng->n_valid;
6550         rte_spinlock_unlock(&cmng->pool_update_sl);
6551         /* Set the statistic memory to the new created pool. */
6552         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
6553                 goto set_alarm;
6554         if (pool->raw_hw)
6555                 /* There is a pool query in progress. */
6556                 goto set_alarm;
6557         pool->raw_hw =
6558                 LIST_FIRST(&sh->cmng.free_stat_raws);
6559         if (!pool->raw_hw)
6560                 /* No free counter statistics raw memory. */
6561                 goto set_alarm;
6562         /*
6563          * Identify the counters released between query trigger and query
6564          * handle more efficiently. The counter released in this gap period
6565          * should wait for a new round of query as the new arrived packets
6566          * will not be taken into account.
6567          */
6568         pool->query_gen++;
6569         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
6570                                                MLX5_COUNTERS_PER_POOL,
6571                                                NULL, NULL,
6572                                                pool->raw_hw->mem_mng->dm->id,
6573                                                (void *)(uintptr_t)
6574                                                pool->raw_hw->data,
6575                                                sh->devx_comp,
6576                                                (uint64_t)(uintptr_t)pool);
6577         if (ret) {
6578                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6579                         " %d", pool->min_dcs->id);
6580                 pool->raw_hw = NULL;
6581                 goto set_alarm;
6582         }
6583         LIST_REMOVE(pool->raw_hw, next);
6584         sh->cmng.pending_queries++;
6585         pool_index++;
6586         if (pool_index >= n_valid)
6587                 pool_index = 0;
6588 set_alarm:
6589         sh->cmng.pool_index = pool_index;
6590         mlx5_set_query_alarm(sh);
6591 }
6592
6593 /**
6594  * Check and callback event for new aged flow in the counter pool
6595  *
6596  * @param[in] sh
6597  *   Pointer to mlx5_dev_ctx_shared object.
6598  * @param[in] pool
6599  *   Pointer to Current counter pool.
6600  */
6601 static void
6602 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6603                    struct mlx5_flow_counter_pool *pool)
6604 {
6605         struct mlx5_priv *priv;
6606         struct mlx5_flow_counter *cnt;
6607         struct mlx5_age_info *age_info;
6608         struct mlx5_age_param *age_param;
6609         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6610         struct mlx5_counter_stats_raw *prev = pool->raw;
6611         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
6612         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
6613         uint16_t expected = AGE_CANDIDATE;
6614         uint32_t i;
6615
6616         pool->time_of_last_age_check = curr_time;
6617         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6618                 cnt = MLX5_POOL_GET_CNT(pool, i);
6619                 age_param = MLX5_CNT_TO_AGE(cnt);
6620                 if (__atomic_load_n(&age_param->state,
6621                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
6622                         continue;
6623                 if (cur->data[i].hits != prev->data[i].hits) {
6624                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
6625                                          __ATOMIC_RELAXED);
6626                         continue;
6627                 }
6628                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
6629                                        time_delta,
6630                                        __ATOMIC_RELAXED) <= age_param->timeout)
6631                         continue;
6632                 /**
6633                  * Hold the lock first, or if between the
6634                  * state AGE_TMOUT and tailq operation the
6635                  * release happened, the release procedure
6636                  * may delete a non-existent tailq node.
6637                  */
6638                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6639                 age_info = GET_PORT_AGE_INFO(priv);
6640                 rte_spinlock_lock(&age_info->aged_sl);
6641                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
6642                                                 AGE_TMOUT, false,
6643                                                 __ATOMIC_RELAXED,
6644                                                 __ATOMIC_RELAXED)) {
6645                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6646                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6647                 }
6648                 rte_spinlock_unlock(&age_info->aged_sl);
6649         }
6650         mlx5_age_event_prepare(sh);
6651 }
6652
6653 /**
6654  * Handler for the HW respond about ready values from an asynchronous batch
6655  * query. This function is probably called by the host thread.
6656  *
6657  * @param[in] sh
6658  *   The pointer to the shared device context.
6659  * @param[in] async_id
6660  *   The Devx async ID.
6661  * @param[in] status
6662  *   The status of the completion.
6663  */
6664 void
6665 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6666                                   uint64_t async_id, int status)
6667 {
6668         struct mlx5_flow_counter_pool *pool =
6669                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6670         struct mlx5_counter_stats_raw *raw_to_free;
6671         uint8_t query_gen = pool->query_gen ^ 1;
6672         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6673         enum mlx5_counter_type cnt_type =
6674                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6675                                 MLX5_COUNTER_TYPE_ORIGIN;
6676
6677         if (unlikely(status)) {
6678                 raw_to_free = pool->raw_hw;
6679         } else {
6680                 raw_to_free = pool->raw;
6681                 if (pool->is_aged)
6682                         mlx5_flow_aging_check(sh, pool);
6683                 rte_spinlock_lock(&pool->sl);
6684                 pool->raw = pool->raw_hw;
6685                 rte_spinlock_unlock(&pool->sl);
6686                 /* Be sure the new raw counters data is updated in memory. */
6687                 rte_io_wmb();
6688                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6689                         rte_spinlock_lock(&cmng->csl[cnt_type]);
6690                         TAILQ_CONCAT(&cmng->counters[cnt_type],
6691                                      &pool->counters[query_gen], next);
6692                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6693                 }
6694         }
6695         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6696         pool->raw_hw = NULL;
6697         sh->cmng.pending_queries--;
6698 }
6699
6700 static int
6701 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
6702                     const struct flow_grp_info *grp_info,
6703                     struct rte_flow_error *error)
6704 {
6705         if (grp_info->transfer && grp_info->external &&
6706             grp_info->fdb_def_rule) {
6707                 if (group == UINT32_MAX)
6708                         return rte_flow_error_set
6709                                                 (error, EINVAL,
6710                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6711                                                  NULL,
6712                                                  "group index not supported");
6713                 *table = group + 1;
6714         } else {
6715                 *table = group;
6716         }
6717         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
6718         return 0;
6719 }
6720
6721 /**
6722  * Translate the rte_flow group index to HW table value.
6723  *
6724  * If tunnel offload is disabled, all group ids converted to flow table
6725  * id using the standard method.
6726  * If tunnel offload is enabled, group id can be converted using the
6727  * standard or tunnel conversion method. Group conversion method
6728  * selection depends on flags in `grp_info` parameter:
6729  * - Internal (grp_info.external == 0) groups conversion uses the
6730  *   standard method.
6731  * - Group ids in JUMP action converted with the tunnel conversion.
6732  * - Group id in rule attribute conversion depends on a rule type and
6733  *   group id value:
6734  *   ** non zero group attributes converted with the tunnel method
6735  *   ** zero group attribute in non-tunnel rule is converted using the
6736  *      standard method - there's only one root table
6737  *   ** zero group attribute in steer tunnel rule is converted with the
6738  *      standard method - single root table
6739  *   ** zero group attribute in match tunnel rule is a special OvS
6740  *      case: that value is used for portability reasons. That group
6741  *      id is converted with the tunnel conversion method.
6742  *
6743  * @param[in] dev
6744  *   Port device
6745  * @param[in] tunnel
6746  *   PMD tunnel offload object
6747  * @param[in] group
6748  *   rte_flow group index value.
6749  * @param[out] table
6750  *   HW table value.
6751  * @param[in] grp_info
6752  *   flags used for conversion
6753  * @param[out] error
6754  *   Pointer to error structure.
6755  *
6756  * @return
6757  *   0 on success, a negative errno value otherwise and rte_errno is set.
6758  */
6759 int
6760 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
6761                          const struct mlx5_flow_tunnel *tunnel,
6762                          uint32_t group, uint32_t *table,
6763                          const struct flow_grp_info *grp_info,
6764                          struct rte_flow_error *error)
6765 {
6766         int ret;
6767         bool standard_translation;
6768
6769         if (!grp_info->skip_scale && grp_info->external &&
6770             group < MLX5_MAX_TABLES_EXTERNAL)
6771                 group *= MLX5_FLOW_TABLE_FACTOR;
6772         if (is_tunnel_offload_active(dev)) {
6773                 standard_translation = !grp_info->external ||
6774                                         grp_info->std_tbl_fix;
6775         } else {
6776                 standard_translation = true;
6777         }
6778         DRV_LOG(DEBUG,
6779                 "port %u group=%#x transfer=%d external=%d fdb_def_rule=%d translate=%s",
6780                 dev->data->port_id, group, grp_info->transfer,
6781                 grp_info->external, grp_info->fdb_def_rule,
6782                 standard_translation ? "STANDARD" : "TUNNEL");
6783         if (standard_translation)
6784                 ret = flow_group_to_table(dev->data->port_id, group, table,
6785                                           grp_info, error);
6786         else
6787                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
6788                                                       table, error);
6789
6790         return ret;
6791 }
6792
6793 /**
6794  * Discover availability of metadata reg_c's.
6795  *
6796  * Iteratively use test flows to check availability.
6797  *
6798  * @param[in] dev
6799  *   Pointer to the Ethernet device structure.
6800  *
6801  * @return
6802  *   0 on success, a negative errno value otherwise and rte_errno is set.
6803  */
6804 int
6805 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6806 {
6807         struct mlx5_priv *priv = dev->data->dev_private;
6808         struct mlx5_dev_config *config = &priv->config;
6809         enum modify_reg idx;
6810         int n = 0;
6811
6812         /* reg_c[0] and reg_c[1] are reserved. */
6813         config->flow_mreg_c[n++] = REG_C_0;
6814         config->flow_mreg_c[n++] = REG_C_1;
6815         /* Discover availability of other reg_c's. */
6816         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6817                 struct rte_flow_attr attr = {
6818                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6819                         .priority = MLX5_FLOW_PRIO_RSVD,
6820                         .ingress = 1,
6821                 };
6822                 struct rte_flow_item items[] = {
6823                         [0] = {
6824                                 .type = RTE_FLOW_ITEM_TYPE_END,
6825                         },
6826                 };
6827                 struct rte_flow_action actions[] = {
6828                         [0] = {
6829                                 .type = (enum rte_flow_action_type)
6830                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6831                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6832                                         .src = REG_C_1,
6833                                         .dst = idx,
6834                                 },
6835                         },
6836                         [1] = {
6837                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6838                                 .conf = &(struct rte_flow_action_jump){
6839                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6840                                 },
6841                         },
6842                         [2] = {
6843                                 .type = RTE_FLOW_ACTION_TYPE_END,
6844                         },
6845                 };
6846                 uint32_t flow_idx;
6847                 struct rte_flow *flow;
6848                 struct rte_flow_error error;
6849
6850                 if (!config->dv_flow_en)
6851                         break;
6852                 /* Create internal flow, validation skips copy action. */
6853                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6854                                             actions, false, &error);
6855                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6856                                       flow_idx);
6857                 if (!flow)
6858                         continue;
6859                 config->flow_mreg_c[n++] = idx;
6860                 flow_list_destroy(dev, NULL, flow_idx);
6861         }
6862         for (; n < MLX5_MREG_C_NUM; ++n)
6863                 config->flow_mreg_c[n] = REG_NON;
6864         return 0;
6865 }
6866
6867 /**
6868  * Dump flow raw hw data to file
6869  *
6870  * @param[in] dev
6871  *    The pointer to Ethernet device.
6872  * @param[in] file
6873  *   A pointer to a file for output.
6874  * @param[out] error
6875  *   Perform verbose error reporting if not NULL. PMDs initialize this
6876  *   structure in case of error only.
6877  * @return
6878  *   0 on success, a nagative value otherwise.
6879  */
6880 int
6881 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6882                    FILE *file,
6883                    struct rte_flow_error *error __rte_unused)
6884 {
6885         struct mlx5_priv *priv = dev->data->dev_private;
6886         struct mlx5_dev_ctx_shared *sh = priv->sh;
6887
6888         if (!priv->config.dv_flow_en) {
6889                 if (fputs("device dv flow disabled\n", file) <= 0)
6890                         return -errno;
6891                 return -ENOTSUP;
6892         }
6893         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6894                                        sh->tx_domain, file);
6895 }
6896
6897 /**
6898  * Get aged-out flows.
6899  *
6900  * @param[in] dev
6901  *   Pointer to the Ethernet device structure.
6902  * @param[in] context
6903  *   The address of an array of pointers to the aged-out flows contexts.
6904  * @param[in] nb_countexts
6905  *   The length of context array pointers.
6906  * @param[out] error
6907  *   Perform verbose error reporting if not NULL. Initialized in case of
6908  *   error only.
6909  *
6910  * @return
6911  *   how many contexts get in success, otherwise negative errno value.
6912  *   if nb_contexts is 0, return the amount of all aged contexts.
6913  *   if nb_contexts is not 0 , return the amount of aged flows reported
6914  *   in the context array.
6915  */
6916 int
6917 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6918                         uint32_t nb_contexts, struct rte_flow_error *error)
6919 {
6920         const struct mlx5_flow_driver_ops *fops;
6921         struct rte_flow_attr attr = { .transfer = 0 };
6922
6923         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6924                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6925                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6926                                                     error);
6927         }
6928         DRV_LOG(ERR,
6929                 "port %u get aged flows is not supported.",
6930                  dev->data->port_id);
6931         return -ENOTSUP;
6932 }
6933
6934 /* Wrapper for driver action_validate op callback */
6935 static int
6936 flow_drv_action_validate(struct rte_eth_dev *dev,
6937                          const struct rte_flow_shared_action_conf *conf,
6938                          const struct rte_flow_action *action,
6939                          const struct mlx5_flow_driver_ops *fops,
6940                          struct rte_flow_error *error)
6941 {
6942         static const char err_msg[] = "shared action validation unsupported";
6943
6944         if (!fops->action_validate) {
6945                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6946                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6947                                    NULL, err_msg);
6948                 return -rte_errno;
6949         }
6950         return fops->action_validate(dev, conf, action, error);
6951 }
6952
6953 /**
6954  * Destroys the shared action by handle.
6955  *
6956  * @param dev
6957  *   Pointer to Ethernet device structure.
6958  * @param[in] action
6959  *   Handle for the shared action to be destroyed.
6960  * @param[out] error
6961  *   Perform verbose error reporting if not NULL. PMDs initialize this
6962  *   structure in case of error only.
6963  *
6964  * @return
6965  *   0 on success, a negative errno value otherwise and rte_errno is set.
6966  *
6967  * @note: wrapper for driver action_create op callback.
6968  */
6969 static int
6970 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
6971                            struct rte_flow_shared_action *action,
6972                            struct rte_flow_error *error)
6973 {
6974         static const char err_msg[] = "shared action destruction unsupported";
6975         struct rte_flow_attr attr = { .transfer = 0 };
6976         const struct mlx5_flow_driver_ops *fops =
6977                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
6978
6979         if (!fops->action_destroy) {
6980                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6981                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6982                                    NULL, err_msg);
6983                 return -rte_errno;
6984         }
6985         return fops->action_destroy(dev, action, error);
6986 }
6987
6988 /* Wrapper for driver action_destroy op callback */
6989 static int
6990 flow_drv_action_update(struct rte_eth_dev *dev,
6991                        struct rte_flow_shared_action *action,
6992                        const void *action_conf,
6993                        const struct mlx5_flow_driver_ops *fops,
6994                        struct rte_flow_error *error)
6995 {
6996         static const char err_msg[] = "shared action update unsupported";
6997
6998         if (!fops->action_update) {
6999                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7000                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7001                                    NULL, err_msg);
7002                 return -rte_errno;
7003         }
7004         return fops->action_update(dev, action, action_conf, error);
7005 }
7006
7007 /* Wrapper for driver action_destroy op callback */
7008 static int
7009 flow_drv_action_query(struct rte_eth_dev *dev,
7010                       const struct rte_flow_shared_action *action,
7011                       void *data,
7012                       const struct mlx5_flow_driver_ops *fops,
7013                       struct rte_flow_error *error)
7014 {
7015         static const char err_msg[] = "shared action query unsupported";
7016
7017         if (!fops->action_query) {
7018                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7019                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7020                                    NULL, err_msg);
7021                 return -rte_errno;
7022         }
7023         return fops->action_query(dev, action, data, error);
7024 }
7025
7026 /**
7027  * Create shared action for reuse in multiple flow rules.
7028  *
7029  * @param dev
7030  *   Pointer to Ethernet device structure.
7031  * @param[in] action
7032  *   Action configuration for shared action creation.
7033  * @param[out] error
7034  *   Perform verbose error reporting if not NULL. PMDs initialize this
7035  *   structure in case of error only.
7036  * @return
7037  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7038  */
7039 static struct rte_flow_shared_action *
7040 mlx5_shared_action_create(struct rte_eth_dev *dev,
7041                           const struct rte_flow_shared_action_conf *conf,
7042                           const struct rte_flow_action *action,
7043                           struct rte_flow_error *error)
7044 {
7045         static const char err_msg[] = "shared action creation unsupported";
7046         struct rte_flow_attr attr = { .transfer = 0 };
7047         const struct mlx5_flow_driver_ops *fops =
7048                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7049
7050         if (flow_drv_action_validate(dev, conf, action, fops, error))
7051                 return NULL;
7052         if (!fops->action_create) {
7053                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7054                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7055                                    NULL, err_msg);
7056                 return NULL;
7057         }
7058         return fops->action_create(dev, conf, action, error);
7059 }
7060
7061 /**
7062  * Updates inplace the shared action configuration pointed by *action* handle
7063  * with the configuration provided as *action* argument.
7064  * The update of the shared action configuration effects all flow rules reusing
7065  * the action via handle.
7066  *
7067  * @param dev
7068  *   Pointer to Ethernet device structure.
7069  * @param[in] shared_action
7070  *   Handle for the shared action to be updated.
7071  * @param[in] action
7072  *   Action specification used to modify the action pointed by handle.
7073  *   *action* should be of same type with the action pointed by the *action*
7074  *   handle argument, otherwise considered as invalid.
7075  * @param[out] error
7076  *   Perform verbose error reporting if not NULL. PMDs initialize this
7077  *   structure in case of error only.
7078  *
7079  * @return
7080  *   0 on success, a negative errno value otherwise and rte_errno is set.
7081  */
7082 static int
7083 mlx5_shared_action_update(struct rte_eth_dev *dev,
7084                 struct rte_flow_shared_action *shared_action,
7085                 const struct rte_flow_action *action,
7086                 struct rte_flow_error *error)
7087 {
7088         struct rte_flow_attr attr = { .transfer = 0 };
7089         const struct mlx5_flow_driver_ops *fops =
7090                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7091         int ret;
7092
7093         ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7094         if (ret)
7095                 return ret;
7096         return flow_drv_action_update(dev, shared_action, action->conf, fops,
7097                                       error);
7098 }
7099
7100 /**
7101  * Query the shared action by handle.
7102  *
7103  * This function allows retrieving action-specific data such as counters.
7104  * Data is gathered by special action which may be present/referenced in
7105  * more than one flow rule definition.
7106  *
7107  * \see RTE_FLOW_ACTION_TYPE_COUNT
7108  *
7109  * @param dev
7110  *   Pointer to Ethernet device structure.
7111  * @param[in] action
7112  *   Handle for the shared action to query.
7113  * @param[in, out] data
7114  *   Pointer to storage for the associated query data type.
7115  * @param[out] error
7116  *   Perform verbose error reporting if not NULL. PMDs initialize this
7117  *   structure in case of error only.
7118  *
7119  * @return
7120  *   0 on success, a negative errno value otherwise and rte_errno is set.
7121  */
7122 static int
7123 mlx5_shared_action_query(struct rte_eth_dev *dev,
7124                          const struct rte_flow_shared_action *action,
7125                          void *data,
7126                          struct rte_flow_error *error)
7127 {
7128         struct rte_flow_attr attr = { .transfer = 0 };
7129         const struct mlx5_flow_driver_ops *fops =
7130                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7131
7132         return flow_drv_action_query(dev, action, data, fops, error);
7133 }
7134
7135 /**
7136  * Destroy all shared actions.
7137  *
7138  * @param dev
7139  *   Pointer to Ethernet device.
7140  *
7141  * @return
7142  *   0 on success, a negative errno value otherwise and rte_errno is set.
7143  */
7144 int
7145 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7146 {
7147         struct rte_flow_error error;
7148         struct mlx5_priv *priv = dev->data->dev_private;
7149         struct mlx5_shared_action_rss *action;
7150         int ret = 0;
7151         uint32_t idx;
7152
7153         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7154                       priv->rss_shared_actions, idx, action, next) {
7155                 ret |= mlx5_shared_action_destroy(dev,
7156                        (struct rte_flow_shared_action *)(uintptr_t)idx, &error);
7157         }
7158         return ret;
7159 }
7160
7161 #ifndef HAVE_MLX5DV_DR
7162 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
7163 #else
7164 #define MLX5_DOMAIN_SYNC_FLOW \
7165         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
7166 #endif
7167
7168 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
7169 {
7170         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
7171         const struct mlx5_flow_driver_ops *fops;
7172         int ret;
7173         struct rte_flow_attr attr = { .transfer = 0 };
7174
7175         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7176         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
7177         if (ret > 0)
7178                 ret = -ret;
7179         return ret;
7180 }
7181
7182 /**
7183  * tunnel offload functionalilty is defined for DV environment only
7184  */
7185 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7186 __extension__
7187 union tunnel_offload_mark {
7188         uint32_t val;
7189         struct {
7190                 uint32_t app_reserve:8;
7191                 uint32_t table_id:15;
7192                 uint32_t transfer:1;
7193                 uint32_t _unused_:8;
7194         };
7195 };
7196
7197 static int
7198 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
7199                              struct rte_flow *flow,
7200                              const struct rte_flow_attr *attr,
7201                              const struct rte_flow_action *app_actions,
7202                              uint32_t flow_idx,
7203                              struct tunnel_default_miss_ctx *ctx,
7204                              struct rte_flow_error *error)
7205 {
7206         struct mlx5_priv *priv = dev->data->dev_private;
7207         struct mlx5_flow *dev_flow;
7208         struct rte_flow_attr miss_attr = *attr;
7209         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
7210         const struct rte_flow_item miss_items[2] = {
7211                 {
7212                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7213                         .spec = NULL,
7214                         .last = NULL,
7215                         .mask = NULL
7216                 },
7217                 {
7218                         .type = RTE_FLOW_ITEM_TYPE_END,
7219                         .spec = NULL,
7220                         .last = NULL,
7221                         .mask = NULL
7222                 }
7223         };
7224         union tunnel_offload_mark mark_id;
7225         struct rte_flow_action_mark miss_mark;
7226         struct rte_flow_action miss_actions[3] = {
7227                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
7228                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
7229         };
7230         const struct rte_flow_action_jump *jump_data;
7231         uint32_t i, flow_table = 0; /* prevent compilation warning */
7232         struct flow_grp_info grp_info = {
7233                 .external = 1,
7234                 .transfer = attr->transfer,
7235                 .fdb_def_rule = !!priv->fdb_def_rule,
7236                 .std_tbl_fix = 0,
7237         };
7238         int ret;
7239
7240         if (!attr->transfer) {
7241                 uint32_t q_size;
7242
7243                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
7244                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
7245                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
7246                                          0, SOCKET_ID_ANY);
7247                 if (!ctx->queue)
7248                         return rte_flow_error_set
7249                                 (error, ENOMEM,
7250                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7251                                 NULL, "invalid default miss RSS");
7252                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7253                 ctx->action_rss.level = 0,
7254                 ctx->action_rss.types = priv->rss_conf.rss_hf,
7255                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
7256                 ctx->action_rss.queue_num = priv->reta_idx_n,
7257                 ctx->action_rss.key = priv->rss_conf.rss_key,
7258                 ctx->action_rss.queue = ctx->queue;
7259                 if (!priv->reta_idx_n || !priv->rxqs_n)
7260                         return rte_flow_error_set
7261                                 (error, EINVAL,
7262                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7263                                 NULL, "invalid port configuration");
7264                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7265                         ctx->action_rss.types = 0;
7266                 for (i = 0; i != priv->reta_idx_n; ++i)
7267                         ctx->queue[i] = (*priv->reta_idx)[i];
7268         } else {
7269                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
7270                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
7271         }
7272         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
7273         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
7274         jump_data = app_actions->conf;
7275         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
7276         miss_attr.group = jump_data->group;
7277         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
7278                                        &flow_table, &grp_info, error);
7279         if (ret)
7280                 return rte_flow_error_set(error, EINVAL,
7281                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7282                                           NULL, "invalid tunnel id");
7283         mark_id.app_reserve = 0;
7284         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
7285         mark_id.transfer = !!attr->transfer;
7286         mark_id._unused_ = 0;
7287         miss_mark.id = mark_id.val;
7288         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
7289                                     miss_items, miss_actions, flow_idx, error);
7290         if (!dev_flow)
7291                 return -rte_errno;
7292         dev_flow->flow = flow;
7293         dev_flow->external = true;
7294         dev_flow->tunnel = tunnel;
7295         /* Subflow object was created, we must include one in the list. */
7296         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7297                       dev_flow->handle, next);
7298         DRV_LOG(DEBUG,
7299                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
7300                 dev->data->port_id, tunnel->app_tunnel.type,
7301                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
7302         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
7303                                   miss_actions, error);
7304         if (!ret)
7305                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
7306                                                   error);
7307
7308         return ret;
7309 }
7310
7311 static const struct mlx5_flow_tbl_data_entry  *
7312 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7313 {
7314         struct mlx5_priv *priv = dev->data->dev_private;
7315         struct mlx5_dev_ctx_shared *sh = priv->sh;
7316         struct mlx5_hlist_entry *he;
7317         union tunnel_offload_mark mbits = { .val = mark };
7318         union mlx5_flow_tbl_key table_key = {
7319                 {
7320                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7321                         .dummy = 0,
7322                         .domain = !!mbits.transfer,
7323                         .direction = 0,
7324                 }
7325         };
7326         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
7327         return he ?
7328                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7329 }
7330
7331 static void
7332 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
7333                                    struct mlx5_hlist_entry *entry)
7334 {
7335         struct mlx5_dev_ctx_shared *sh = list->ctx;
7336         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7337
7338         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7339                         tunnel_flow_tbl_to_id(tte->flow_table));
7340         mlx5_free(tte);
7341 }
7342
7343 static struct mlx5_hlist_entry *
7344 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
7345                                    uint64_t key __rte_unused,
7346                                    void *ctx __rte_unused)
7347 {
7348         struct mlx5_dev_ctx_shared *sh = list->ctx;
7349         struct tunnel_tbl_entry *tte;
7350
7351         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7352                           sizeof(*tte), 0,
7353                           SOCKET_ID_ANY);
7354         if (!tte)
7355                 goto err;
7356         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7357                           &tte->flow_table);
7358         if (tte->flow_table >= MLX5_MAX_TABLES) {
7359                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
7360                         tte->flow_table);
7361                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7362                                 tte->flow_table);
7363                 goto err;
7364         } else if (!tte->flow_table) {
7365                 goto err;
7366         }
7367         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7368         return &tte->hash;
7369 err:
7370         if (tte)
7371                 mlx5_free(tte);
7372         return NULL;
7373 }
7374
7375 static uint32_t
7376 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7377                                 const struct mlx5_flow_tunnel *tunnel,
7378                                 uint32_t group, uint32_t *table,
7379                                 struct rte_flow_error *error)
7380 {
7381         struct mlx5_hlist_entry *he;
7382         struct tunnel_tbl_entry *tte;
7383         union tunnel_tbl_key key = {
7384                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7385                 .group = group
7386         };
7387         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7388         struct mlx5_hlist *group_hash;
7389
7390         group_hash = tunnel ? tunnel->groups : thub->groups;
7391         he = mlx5_hlist_register(group_hash, key.val, NULL);
7392         if (!he)
7393                 return rte_flow_error_set(error, EINVAL,
7394                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7395                                           NULL,
7396                                           "tunnel group index not supported");
7397         tte = container_of(he, typeof(*tte), hash);
7398         *table = tte->flow_table;
7399         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7400                 dev->data->port_id, key.tunnel_id, group, *table);
7401         return 0;
7402 }
7403
7404 static void
7405 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
7406                       struct mlx5_flow_tunnel *tunnel)
7407 {
7408         struct mlx5_priv *priv = dev->data->dev_private;
7409         struct mlx5_indexed_pool *ipool;
7410
7411         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
7412                 dev->data->port_id, tunnel->tunnel_id);
7413         mlx5_hlist_destroy(tunnel->groups);
7414         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7415         mlx5_ipool_free(ipool, tunnel->tunnel_id);
7416 }
7417
7418 static struct mlx5_flow_tunnel *
7419 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
7420 {
7421         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7422         struct mlx5_flow_tunnel *tun;
7423
7424         LIST_FOREACH(tun, &thub->tunnels, chain) {
7425                 if (tun->tunnel_id == id)
7426                         break;
7427         }
7428
7429         return tun;
7430 }
7431
7432 static struct mlx5_flow_tunnel *
7433 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
7434                           const struct rte_flow_tunnel *app_tunnel)
7435 {
7436         struct mlx5_priv *priv = dev->data->dev_private;
7437         struct mlx5_indexed_pool *ipool;
7438         struct mlx5_flow_tunnel *tunnel;
7439         uint32_t id;
7440
7441         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7442         tunnel = mlx5_ipool_zmalloc(ipool, &id);
7443         if (!tunnel)
7444                 return NULL;
7445         if (id >= MLX5_MAX_TUNNELS) {
7446                 mlx5_ipool_free(ipool, id);
7447                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
7448                 return NULL;
7449         }
7450         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
7451                                            mlx5_flow_tunnel_grp2tbl_create_cb,
7452                                            NULL,
7453                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
7454         if (!tunnel->groups) {
7455                 mlx5_ipool_free(ipool, id);
7456                 return NULL;
7457         }
7458         tunnel->groups->ctx = priv->sh;
7459         /* initiate new PMD tunnel */
7460         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
7461         tunnel->tunnel_id = id;
7462         tunnel->action.type = (typeof(tunnel->action.type))
7463                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
7464         tunnel->action.conf = tunnel;
7465         tunnel->item.type = (typeof(tunnel->item.type))
7466                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
7467         tunnel->item.spec = tunnel;
7468         tunnel->item.last = NULL;
7469         tunnel->item.mask = NULL;
7470
7471         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
7472                 dev->data->port_id, tunnel->tunnel_id);
7473
7474         return tunnel;
7475 }
7476
7477 static int
7478 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
7479                      const struct rte_flow_tunnel *app_tunnel,
7480                      struct mlx5_flow_tunnel **tunnel)
7481 {
7482         int ret;
7483         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7484         struct mlx5_flow_tunnel *tun;
7485
7486         rte_spinlock_lock(&thub->sl);
7487         LIST_FOREACH(tun, &thub->tunnels, chain) {
7488                 if (!memcmp(app_tunnel, &tun->app_tunnel,
7489                             sizeof(*app_tunnel))) {
7490                         *tunnel = tun;
7491                         ret = 0;
7492                         break;
7493                 }
7494         }
7495         if (!tun) {
7496                 tun = mlx5_flow_tunnel_allocate(dev, app_tunnel);
7497                 if (tun) {
7498                         LIST_INSERT_HEAD(&thub->tunnels, tun, chain);
7499                         *tunnel = tun;
7500                 } else {
7501                         ret = -ENOMEM;
7502                 }
7503         }
7504         rte_spinlock_unlock(&thub->sl);
7505         if (tun)
7506                 __atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
7507
7508         return ret;
7509 }
7510
7511 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
7512 {
7513         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
7514
7515         if (!thub)
7516                 return;
7517         if (!LIST_EMPTY(&thub->tunnels))
7518                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
7519         mlx5_hlist_destroy(thub->groups);
7520         mlx5_free(thub);
7521 }
7522
7523 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
7524 {
7525         int err;
7526         struct mlx5_flow_tunnel_hub *thub;
7527
7528         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
7529                            0, SOCKET_ID_ANY);
7530         if (!thub)
7531                 return -ENOMEM;
7532         LIST_INIT(&thub->tunnels);
7533         rte_spinlock_init(&thub->sl);
7534         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
7535                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
7536                                          NULL,
7537                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
7538         if (!thub->groups) {
7539                 err = -rte_errno;
7540                 goto err;
7541         }
7542         thub->groups->ctx = sh;
7543         sh->tunnel_hub = thub;
7544
7545         return 0;
7546
7547 err:
7548         if (thub->groups)
7549                 mlx5_hlist_destroy(thub->groups);
7550         if (thub)
7551                 mlx5_free(thub);
7552         return err;
7553 }
7554
7555 static inline bool
7556 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
7557                           struct rte_flow_tunnel *tunnel,
7558                           const char *err_msg)
7559 {
7560         err_msg = NULL;
7561         if (!is_tunnel_offload_active(dev)) {
7562                 err_msg = "tunnel offload was not activated";
7563                 goto out;
7564         } else if (!tunnel) {
7565                 err_msg = "no application tunnel";
7566                 goto out;
7567         }
7568
7569         switch (tunnel->type) {
7570         default:
7571                 err_msg = "unsupported tunnel type";
7572                 goto out;
7573         case RTE_FLOW_ITEM_TYPE_VXLAN:
7574                 break;
7575         }
7576
7577 out:
7578         return !err_msg;
7579 }
7580
7581 static int
7582 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
7583                     struct rte_flow_tunnel *app_tunnel,
7584                     struct rte_flow_action **actions,
7585                     uint32_t *num_of_actions,
7586                     struct rte_flow_error *error)
7587 {
7588         int ret;
7589         struct mlx5_flow_tunnel *tunnel;
7590         const char *err_msg = NULL;
7591         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7592
7593         if (!verdict)
7594                 return rte_flow_error_set(error, EINVAL,
7595                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7596                                           err_msg);
7597         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7598         if (ret < 0) {
7599                 return rte_flow_error_set(error, ret,
7600                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7601                                           "failed to initialize pmd tunnel");
7602         }
7603         *actions = &tunnel->action;
7604         *num_of_actions = 1;
7605         return 0;
7606 }
7607
7608 static int
7609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
7610                        struct rte_flow_tunnel *app_tunnel,
7611                        struct rte_flow_item **items,
7612                        uint32_t *num_of_items,
7613                        struct rte_flow_error *error)
7614 {
7615         int ret;
7616         struct mlx5_flow_tunnel *tunnel;
7617         const char *err_msg = NULL;
7618         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7619
7620         if (!verdict)
7621                 return rte_flow_error_set(error, EINVAL,
7622                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7623                                           err_msg);
7624         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7625         if (ret < 0) {
7626                 return rte_flow_error_set(error, ret,
7627                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7628                                           "failed to initialize pmd tunnel");
7629         }
7630         *items = &tunnel->item;
7631         *num_of_items = 1;
7632         return 0;
7633 }
7634 static int
7635 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
7636                               struct rte_flow_item *pmd_items,
7637                               uint32_t num_items, struct rte_flow_error *err)
7638 {
7639         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7640         struct mlx5_flow_tunnel *tun;
7641
7642         rte_spinlock_lock(&thub->sl);
7643         LIST_FOREACH(tun, &thub->tunnels, chain) {
7644                 if (&tun->item == pmd_items) {
7645                         LIST_REMOVE(tun, chain);
7646                         break;
7647                 }
7648         }
7649         rte_spinlock_unlock(&thub->sl);
7650         if (!tun || num_items != 1)
7651                 return rte_flow_error_set(err, EINVAL,
7652                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7653                                           "invalid argument");
7654         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
7655                 mlx5_flow_tunnel_free(dev, tun);
7656         return 0;
7657 }
7658
7659 static int
7660 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
7661                                 struct rte_flow_action *pmd_actions,
7662                                 uint32_t num_actions,
7663                                 struct rte_flow_error *err)
7664 {
7665         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7666         struct mlx5_flow_tunnel *tun;
7667
7668         rte_spinlock_lock(&thub->sl);
7669         LIST_FOREACH(tun, &thub->tunnels, chain) {
7670                 if (&tun->action == pmd_actions) {
7671                         LIST_REMOVE(tun, chain);
7672                         break;
7673                 }
7674         }
7675         rte_spinlock_unlock(&thub->sl);
7676         if (!tun || num_actions != 1)
7677                 return rte_flow_error_set(err, EINVAL,
7678                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7679                                           "invalid argument");
7680         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
7681                 mlx5_flow_tunnel_free(dev, tun);
7682
7683         return 0;
7684 }
7685
7686 static int
7687 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
7688                                   struct rte_mbuf *m,
7689                                   struct rte_flow_restore_info *info,
7690                                   struct rte_flow_error *err)
7691 {
7692         uint64_t ol_flags = m->ol_flags;
7693         const struct mlx5_flow_tbl_data_entry *tble;
7694         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
7695
7696         if ((ol_flags & mask) != mask)
7697                 goto err;
7698         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
7699         if (!tble) {
7700                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
7701                         dev->data->port_id, m->hash.fdir.hi);
7702                 goto err;
7703         }
7704         MLX5_ASSERT(tble->tunnel);
7705         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
7706         info->group_id = tble->group_id;
7707         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
7708                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
7709                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
7710
7711         return 0;
7712
7713 err:
7714         return rte_flow_error_set(err, EINVAL,
7715                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7716                                   "failed to get restore info");
7717 }
7718
7719 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
7720 static int
7721 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
7722                            __rte_unused struct rte_flow_tunnel *app_tunnel,
7723                            __rte_unused struct rte_flow_action **actions,
7724                            __rte_unused uint32_t *num_of_actions,
7725                            __rte_unused struct rte_flow_error *error)
7726 {
7727         return -ENOTSUP;
7728 }
7729
7730 static int
7731 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
7732                        __rte_unused struct rte_flow_tunnel *app_tunnel,
7733                        __rte_unused struct rte_flow_item **items,
7734                        __rte_unused uint32_t *num_of_items,
7735                        __rte_unused struct rte_flow_error *error)
7736 {
7737         return -ENOTSUP;
7738 }
7739
7740 static int
7741 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
7742                               __rte_unused struct rte_flow_item *pmd_items,
7743                               __rte_unused uint32_t num_items,
7744                               __rte_unused struct rte_flow_error *err)
7745 {
7746         return -ENOTSUP;
7747 }
7748
7749 static int
7750 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
7751                                 __rte_unused struct rte_flow_action *pmd_action,
7752                                 __rte_unused uint32_t num_actions,
7753                                 __rte_unused struct rte_flow_error *err)
7754 {
7755         return -ENOTSUP;
7756 }
7757
7758 static int
7759 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
7760                                   __rte_unused struct rte_mbuf *m,
7761                                   __rte_unused struct rte_flow_restore_info *i,
7762                                   __rte_unused struct rte_flow_error *err)
7763 {
7764         return -ENOTSUP;
7765 }
7766
7767 static int
7768 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
7769                              __rte_unused struct rte_flow *flow,
7770                              __rte_unused const struct rte_flow_attr *attr,
7771                              __rte_unused const struct rte_flow_action *actions,
7772                              __rte_unused uint32_t flow_idx,
7773                              __rte_unused struct tunnel_default_miss_ctx *ctx,
7774                              __rte_unused struct rte_flow_error *error)
7775 {
7776         return -ENOTSUP;
7777 }
7778
7779 static struct mlx5_flow_tunnel *
7780 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
7781                     __rte_unused uint32_t id)
7782 {
7783         return NULL;
7784 }
7785
7786 static void
7787 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
7788                       __rte_unused struct mlx5_flow_tunnel *tunnel)
7789 {
7790 }
7791
7792 static uint32_t
7793 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
7794                                 __rte_unused const struct mlx5_flow_tunnel *t,
7795                                 __rte_unused uint32_t group,
7796                                 __rte_unused uint32_t *table,
7797                                 struct rte_flow_error *error)
7798 {
7799         return rte_flow_error_set(error, ENOTSUP,
7800                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7801                                   "tunnel offload requires DV support");
7802 }
7803
7804 void
7805 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
7806                         __rte_unused  uint16_t port_id)
7807 {
7808 }
7809 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
7810