drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              struct tunnel_default_miss_ctx *ctx,
  54                              struct rte_flow_error *error);
  55 static struct mlx5_flow_tunnel *
  56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  57 static void
  58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  59 static uint32_t
  60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  61                                 const struct mlx5_flow_tunnel *tunnel,
  62                                 uint32_t group, uint32_t *table,
  63                                 struct rte_flow_error *error);
  64
  65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  66 static void mlx5_flow_pop_thread_workspace(void);
  67
  68
  69 /** Device flow drivers. */
  70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  71
  72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  73
  74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  75         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  77         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  78 #endif
  79         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  80         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  81 };
  82
  83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  85         (const int []){ \
  86                 __VA_ARGS__, 0, \
  87         }
  88
  89 /** Node object of input graph for mlx5_flow_expand_rss(). */
  90 struct mlx5_flow_expand_node {
  91         const int *const next;
  92         /**<
  93          * List of next node indexes. Index 0 is interpreted as a terminator.
  94          */
  95         const enum rte_flow_item_type type;
  96         /**< Pattern item type of current node. */
  97         uint64_t rss_types;
  98         /**<
  99          * RSS types bit-field associated with this node
 100          * (see ETH_RSS_* definitions).
 101          */
 102 };
 103
 104 /** Object returned by mlx5_flow_expand_rss(). */
 105 struct mlx5_flow_expand_rss {
 106         uint32_t entries;
 107         /**< Number of entries @p patterns and @p priorities. */
 108         struct {
 109                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 110                 uint32_t priority; /**< Priority offset for each expansion. */
 111         } entry[];
 112 };
 113
 114 static enum rte_flow_item_type
 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 116 {
 117         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 118         uint16_t ether_type = 0;
 119         uint16_t ether_type_m;
 120         uint8_t ip_next_proto = 0;
 121         uint8_t ip_next_proto_m;
 122
 123         if (item == NULL || item->spec == NULL)
 124                 return ret;
 125         switch (item->type) {
 126         case RTE_FLOW_ITEM_TYPE_ETH:
 127                 if (item->mask)
 128                         ether_type_m = ((const struct rte_flow_item_eth *)
 129                                                 (item->mask))->type;
 130                 else
 131                         ether_type_m = rte_flow_item_eth_mask.type;
 132                 if (ether_type_m != RTE_BE16(0xFFFF))
 133                         break;
 134                 ether_type = ((const struct rte_flow_item_eth *)
 135                                 (item->spec))->type;
 136                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 139                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 140                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 141                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 142                 else
 143                         ret = RTE_FLOW_ITEM_TYPE_END;
 144                 break;
 145         case RTE_FLOW_ITEM_TYPE_VLAN:
 146                 if (item->mask)
 147                         ether_type_m = ((const struct rte_flow_item_vlan *)
 148                                                 (item->mask))->inner_type;
 149                 else
 150                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 151                 if (ether_type_m != RTE_BE16(0xFFFF))
 152                         break;
 153                 ether_type = ((const struct rte_flow_item_vlan *)
 154                                 (item->spec))->inner_type;
 155                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 156                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 157                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 159                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 160                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV4:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 167                                         (item->mask))->hdr.next_proto_id;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 174                                 (item->spec))->hdr.next_proto_id;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         case RTE_FLOW_ITEM_TYPE_IPV6:
 187                 if (item->mask)
 188                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 189                                                 (item->mask))->hdr.proto;
 190                 else
 191                         ip_next_proto_m =
 192                                 rte_flow_item_ipv6_mask.hdr.proto;
 193                 if (ip_next_proto_m != 0xFF)
 194                         break;
 195                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 196                                 (item->spec))->hdr.proto;
 197                 if (ip_next_proto == IPPROTO_UDP)
 198                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 199                 else if (ip_next_proto == IPPROTO_TCP)
 200                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 201                 else if (ip_next_proto == IPPROTO_IP)
 202                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 203                 else if (ip_next_proto == IPPROTO_IPV6)
 204                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 205                 else
 206                         ret = RTE_FLOW_ITEM_TYPE_END;
 207                 break;
 208         default:
 209                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 210                 break;
 211         }
 212         return ret;
 213 }
 214
 215 /**
 216  * Expand RSS flows into several possible flows according to the RSS hash
 217  * fields requested and the driver capabilities.
 218  *
 219  * @param[out] buf
 220  *   Buffer to store the result expansion.
 221  * @param[in] size
 222  *   Buffer size in bytes. If 0, @p buf can be NULL.
 223  * @param[in] pattern
 224  *   User flow pattern.
 225  * @param[in] types
 226  *   RSS types to expand (see ETH_RSS_* definitions).
 227  * @param[in] graph
 228  *   Input graph to expand @p pattern according to @p types.
 229  * @param[in] graph_root_index
 230  *   Index of root node in @p graph, typically 0.
 231  *
 232  * @return
 233  *   A positive value representing the size of @p buf in bytes regardless of
 234  *   @p size on success, a negative errno value otherwise and rte_errno is
 235  *   set, the following errors are defined:
 236  *
 237  *   -E2BIG: graph-depth @p graph is too deep.
 238  */
 239 static int
 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 241                      const struct rte_flow_item *pattern, uint64_t types,
 242                      const struct mlx5_flow_expand_node graph[],
 243                      int graph_root_index)
 244 {
 245         const int elt_n = 8;
 246         const struct rte_flow_item *item;
 247         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 248         const int *next_node;
 249         const int *stack[elt_n];
 250         int stack_pos = 0;
 251         struct rte_flow_item flow_items[elt_n];
 252         unsigned int i;
 253         size_t lsize;
 254         size_t user_pattern_size = 0;
 255         void *addr = NULL;
 256         const struct mlx5_flow_expand_node *next = NULL;
 257         struct rte_flow_item missed_item;
 258         int missed = 0;
 259         int elt = 0;
 260         const struct rte_flow_item *last_item = NULL;
 261
 262         memset(&missed_item, 0, sizeof(missed_item));
 263         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 264                 elt_n * sizeof(buf->entry[0]);
 265         if (lsize <= size) {
 266                 buf->entry[0].priority = 0;
 267                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 268                 buf->entries = 0;
 269                 addr = buf->entry[0].pattern;
 270         }
 271         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 272                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 273                         last_item = item;
 274                 for (i = 0; node->next && node->next[i]; ++i) {
 275                         next = &graph[node->next[i]];
 276                         if (next->type == item->type)
 277                                 break;
 278                 }
 279                 if (next)
 280                         node = next;
 281                 user_pattern_size += sizeof(*item);
 282         }
 283         user_pattern_size += sizeof(*item); /* Handle END item. */
 284         lsize += user_pattern_size;
 285         /* Copy the user pattern in the first entry of the buffer. */
 286         if (lsize <= size) {
 287                 rte_memcpy(addr, pattern, user_pattern_size);
 288                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 289                 buf->entries = 1;
 290         }
 291         /* Start expanding. */
 292         memset(flow_items, 0, sizeof(flow_items));
 293         user_pattern_size -= sizeof(*item);
 294         /*
 295          * Check if the last valid item has spec set, need complete pattern,
 296          * and the pattern can be used for expansion.
 297          */
 298         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 299         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 300                 /* Item type END indicates expansion is not required. */
 301                 return lsize;
 302         }
 303         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 304                 next = NULL;
 305                 missed = 1;
 306                 for (i = 0; node->next && node->next[i]; ++i) {
 307                         next = &graph[node->next[i]];
 308                         if (next->type == missed_item.type) {
 309                                 flow_items[0].type = missed_item.type;
 310                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 311                                 break;
 312                         }
 313                         next = NULL;
 314                 }
 315         }
 316         if (next && missed) {
 317                 elt = 2; /* missed item + item end. */
 318                 node = next;
 319                 lsize += elt * sizeof(*item) + user_pattern_size;
 320                 if ((node->rss_types & types) && lsize <= size) {
 321                         buf->entry[buf->entries].priority = 1;
 322                         buf->entry[buf->entries].pattern = addr;
 323                         buf->entries++;
 324                         rte_memcpy(addr, buf->entry[0].pattern,
 325                                    user_pattern_size);
 326                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 327                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 328                         addr = (void *)(((uintptr_t)addr) +
 329                                         elt * sizeof(*item));
 330                 }
 331         }
 332         memset(flow_items, 0, sizeof(flow_items));
 333         next_node = node->next;
 334         stack[stack_pos] = next_node;
 335         node = next_node ? &graph[*next_node] : NULL;
 336         while (node) {
 337                 flow_items[stack_pos].type = node->type;
 338                 if (node->rss_types & types) {
 339                         /*
 340                          * compute the number of items to copy from the
 341                          * expansion and copy it.
 342                          * When the stack_pos is 0, there are 1 element in it,
 343                          * plus the addition END item.
 344                          */
 345                         elt = stack_pos + 2;
 346                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 347                         lsize += elt * sizeof(*item) + user_pattern_size;
 348                         if (lsize <= size) {
 349                                 size_t n = elt * sizeof(*item);
 350
 351                                 buf->entry[buf->entries].priority =
 352                                         stack_pos + 1 + missed;
 353                                 buf->entry[buf->entries].pattern = addr;
 354                                 buf->entries++;
 355                                 rte_memcpy(addr, buf->entry[0].pattern,
 356                                            user_pattern_size);
 357                                 addr = (void *)(((uintptr_t)addr) +
 358                                                 user_pattern_size);
 359                                 rte_memcpy(addr, &missed_item,
 360                                            missed * sizeof(*item));
 361                                 addr = (void *)(((uintptr_t)addr) +
 362                                         missed * sizeof(*item));
 363                                 rte_memcpy(addr, flow_items, n);
 364                                 addr = (void *)(((uintptr_t)addr) + n);
 365                         }
 366                 }
 367                 /* Go deeper. */
 368                 if (node->next) {
 369                         next_node = node->next;
 370                         if (stack_pos++ == elt_n) {
 371                                 rte_errno = E2BIG;
 372                                 return -rte_errno;
 373                         }
 374                         stack[stack_pos] = next_node;
 375                 } else if (*(next_node + 1)) {
 376                         /* Follow up with the next possibility. */
 377                         ++next_node;
 378                 } else {
 379                         /* Move to the next path. */
 380                         if (stack_pos)
 381                                 next_node = stack[--stack_pos];
 382                         next_node++;
 383                         stack[stack_pos] = next_node;
 384                 }
 385                 node = *next_node ? &graph[*next_node] : NULL;
 386         };
 387         /* no expanded flows but we have missed item, create one rule for it */
 388         if (buf->entries == 1 && missed != 0) {
 389                 elt = 2;
 390                 lsize += elt * sizeof(*item) + user_pattern_size;
 391                 if (lsize <= size) {
 392                         buf->entry[buf->entries].priority = 1;
 393                         buf->entry[buf->entries].pattern = addr;
 394                         buf->entries++;
 395                         flow_items[0].type = missed_item.type;
 396                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 397                         rte_memcpy(addr, buf->entry[0].pattern,
 398                                    user_pattern_size);
 399                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 400                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 401                 }
 402         }
 403         return lsize;
 404 }
 405
 406 enum mlx5_expansion {
 407         MLX5_EXPANSION_ROOT,
 408         MLX5_EXPANSION_ROOT_OUTER,
 409         MLX5_EXPANSION_ROOT_ETH_VLAN,
 410         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 411         MLX5_EXPANSION_OUTER_ETH,
 412         MLX5_EXPANSION_OUTER_ETH_VLAN,
 413         MLX5_EXPANSION_OUTER_VLAN,
 414         MLX5_EXPANSION_OUTER_IPV4,
 415         MLX5_EXPANSION_OUTER_IPV4_UDP,
 416         MLX5_EXPANSION_OUTER_IPV4_TCP,
 417         MLX5_EXPANSION_OUTER_IPV6,
 418         MLX5_EXPANSION_OUTER_IPV6_UDP,
 419         MLX5_EXPANSION_OUTER_IPV6_TCP,
 420         MLX5_EXPANSION_VXLAN,
 421         MLX5_EXPANSION_VXLAN_GPE,
 422         MLX5_EXPANSION_GRE,
 423         MLX5_EXPANSION_MPLS,
 424         MLX5_EXPANSION_ETH,
 425         MLX5_EXPANSION_ETH_VLAN,
 426         MLX5_EXPANSION_VLAN,
 427         MLX5_EXPANSION_IPV4,
 428         MLX5_EXPANSION_IPV4_UDP,
 429         MLX5_EXPANSION_IPV4_TCP,
 430         MLX5_EXPANSION_IPV6,
 431         MLX5_EXPANSION_IPV6_UDP,
 432         MLX5_EXPANSION_IPV6_TCP,
 433 };
 434
 435 /** Supported expansion of items. */
 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 437         [MLX5_EXPANSION_ROOT] = {
 438                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 439                                                   MLX5_EXPANSION_IPV4,
 440                                                   MLX5_EXPANSION_IPV6),
 441                 .type = RTE_FLOW_ITEM_TYPE_END,
 442         },
 443         [MLX5_EXPANSION_ROOT_OUTER] = {
 444                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 445                                                   MLX5_EXPANSION_OUTER_IPV4,
 446                                                   MLX5_EXPANSION_OUTER_IPV6),
 447                 .type = RTE_FLOW_ITEM_TYPE_END,
 448         },
 449         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 450                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 451                 .type = RTE_FLOW_ITEM_TYPE_END,
 452         },
 453         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 454                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 455                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 456                 .type = RTE_FLOW_ITEM_TYPE_END,
 457         },
 458         [MLX5_EXPANSION_OUTER_ETH] = {
 459                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 460                                                   MLX5_EXPANSION_OUTER_IPV6,
 461                                                   MLX5_EXPANSION_MPLS),
 462                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 463                 .rss_types = 0,
 464         },
 465         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 466                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 467                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 468                 .rss_types = 0,
 469         },
 470         [MLX5_EXPANSION_OUTER_VLAN] = {
 471                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 472                                                   MLX5_EXPANSION_OUTER_IPV6),
 473                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 474         },
 475         [MLX5_EXPANSION_OUTER_IPV4] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 477                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 478                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 479                          MLX5_EXPANSION_GRE,
 480                          MLX5_EXPANSION_IPV4,
 481                          MLX5_EXPANSION_IPV6),
 482                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 483                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 484                         ETH_RSS_NONFRAG_IPV4_OTHER,
 485         },
 486         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 487                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 488                                                   MLX5_EXPANSION_VXLAN_GPE),
 489                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 490                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 493                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 494                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 495         },
 496         [MLX5_EXPANSION_OUTER_IPV6] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 498                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 499                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 500                          MLX5_EXPANSION_IPV4,
 501                          MLX5_EXPANSION_IPV6),
 502                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 503                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 504                         ETH_RSS_NONFRAG_IPV6_OTHER,
 505         },
 506         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 507                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 508                                                   MLX5_EXPANSION_VXLAN_GPE),
 509                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 510                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 511         },
 512         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 513                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 514                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 515         },
 516         [MLX5_EXPANSION_VXLAN] = {
 517                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 518                                                   MLX5_EXPANSION_IPV4,
 519                                                   MLX5_EXPANSION_IPV6),
 520                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 521         },
 522         [MLX5_EXPANSION_VXLAN_GPE] = {
 523                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 524                                                   MLX5_EXPANSION_IPV4,
 525                                                   MLX5_EXPANSION_IPV6),
 526                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 527         },
 528         [MLX5_EXPANSION_GRE] = {
 529                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 530                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 531         },
 532         [MLX5_EXPANSION_MPLS] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 534                                                   MLX5_EXPANSION_IPV6),
 535                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 536         },
 537         [MLX5_EXPANSION_ETH] = {
 538                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 539                                                   MLX5_EXPANSION_IPV6),
 540                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 541         },
 542         [MLX5_EXPANSION_ETH_VLAN] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 544                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 545         },
 546         [MLX5_EXPANSION_VLAN] = {
 547                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 548                                                   MLX5_EXPANSION_IPV6),
 549                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 550         },
 551         [MLX5_EXPANSION_IPV4] = {
 552                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 553                                                   MLX5_EXPANSION_IPV4_TCP),
 554                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 555                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 556                         ETH_RSS_NONFRAG_IPV4_OTHER,
 557         },
 558         [MLX5_EXPANSION_IPV4_UDP] = {
 559                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 560                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 561         },
 562         [MLX5_EXPANSION_IPV4_TCP] = {
 563                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 564                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 565         },
 566         [MLX5_EXPANSION_IPV6] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 568                                                   MLX5_EXPANSION_IPV6_TCP),
 569                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 570                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 571                         ETH_RSS_NONFRAG_IPV6_OTHER,
 572         },
 573         [MLX5_EXPANSION_IPV6_UDP] = {
 574                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 575                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 576         },
 577         [MLX5_EXPANSION_IPV6_TCP] = {
 578                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 579                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 580         },
 581 };
 582
 583 static struct rte_flow_shared_action *
 584 mlx5_shared_action_create(struct rte_eth_dev *dev,
 585                           const struct rte_flow_shared_action_conf *conf,
 586                           const struct rte_flow_action *action,
 587                           struct rte_flow_error *error);
 588 static int mlx5_shared_action_destroy
 589                                 (struct rte_eth_dev *dev,
 590                                  struct rte_flow_shared_action *shared_action,
 591                                  struct rte_flow_error *error);
 592 static int mlx5_shared_action_update
 593                                 (struct rte_eth_dev *dev,
 594                                  struct rte_flow_shared_action *shared_action,
 595                                  const struct rte_flow_action *action,
 596                                  struct rte_flow_error *error);
 597 static int mlx5_shared_action_query
 598                                 (struct rte_eth_dev *dev,
 599                                  const struct rte_flow_shared_action *action,
 600                                  void *data,
 601                                  struct rte_flow_error *error);
 602 static int
 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 604                     struct rte_flow_tunnel *app_tunnel,
 605                     struct rte_flow_action **actions,
 606                     uint32_t *num_of_actions,
 607                     struct rte_flow_error *error);
 608 static int
 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 610                        struct rte_flow_tunnel *app_tunnel,
 611                        struct rte_flow_item **items,
 612                        uint32_t *num_of_items,
 613                        struct rte_flow_error *error);
 614 static int
 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 616                               struct rte_flow_item *pmd_items,
 617                               uint32_t num_items, struct rte_flow_error *err);
 618 static int
 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 620                                 struct rte_flow_action *pmd_actions,
 621                                 uint32_t num_actions,
 622                                 struct rte_flow_error *err);
 623 static int
 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 625                                   struct rte_mbuf *m,
 626                                   struct rte_flow_restore_info *info,
 627                                   struct rte_flow_error *err);
 628
 629 static const struct rte_flow_ops mlx5_flow_ops = {
 630         .validate = mlx5_flow_validate,
 631         .create = mlx5_flow_create,
 632         .destroy = mlx5_flow_destroy,
 633         .flush = mlx5_flow_flush,
 634         .isolate = mlx5_flow_isolate,
 635         .query = mlx5_flow_query,
 636         .dev_dump = mlx5_flow_dev_dump,
 637         .get_aged_flows = mlx5_flow_get_aged_flows,
 638         .shared_action_create = mlx5_shared_action_create,
 639         .shared_action_destroy = mlx5_shared_action_destroy,
 640         .shared_action_update = mlx5_shared_action_update,
 641         .shared_action_query = mlx5_shared_action_query,
 642         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 643         .tunnel_match = mlx5_flow_tunnel_match,
 644         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 645         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 646         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 647 };
 648
 649 /* Tunnel information. */
 650 struct mlx5_flow_tunnel_info {
 651         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 652         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 653 };
 654
 655 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 656         {
 657                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 658                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 659         },
 660         {
 661                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 662                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 663         },
 664         {
 665                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 666                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 667         },
 668         {
 669                 .tunnel = MLX5_FLOW_LAYER_GRE,
 670                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 671         },
 672         {
 673                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 674                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 675         },
 676         {
 677                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 678                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 679         },
 680         {
 681                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 682                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 683         },
 684         {
 685                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 686                 .ptype = RTE_PTYPE_TUNNEL_IP,
 687         },
 688         {
 689                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 690                 .ptype = RTE_PTYPE_TUNNEL_IP,
 691         },
 692         {
 693                 .tunnel = MLX5_FLOW_LAYER_GTP,
 694                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 695         },
 696 };
 697
 698 /* Key of thread specific flow workspace data. */
 699 static pthread_key_t key_workspace;
 700
 701 /* Thread specific flow workspace data once initialization data. */
 702 static pthread_once_t key_workspace_init;
 703
 704
 705 /**
 706  * Translate tag ID to register.
 707  *
 708  * @param[in] dev
 709  *   Pointer to the Ethernet device structure.
 710  * @param[in] feature
 711  *   The feature that request the register.
 712  * @param[in] id
 713  *   The request register ID.
 714  * @param[out] error
 715  *   Error description in case of any.
 716  *
 717  * @return
 718  *   The request register on success, a negative errno
 719  *   value otherwise and rte_errno is set.
 720  */
 721 int
 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 723                      enum mlx5_feature_name feature,
 724                      uint32_t id,
 725                      struct rte_flow_error *error)
 726 {
 727         struct mlx5_priv *priv = dev->data->dev_private;
 728         struct mlx5_dev_config *config = &priv->config;
 729         enum modify_reg start_reg;
 730         bool skip_mtr_reg = false;
 731
 732         switch (feature) {
 733         case MLX5_HAIRPIN_RX:
 734                 return REG_B;
 735         case MLX5_HAIRPIN_TX:
 736                 return REG_A;
 737         case MLX5_METADATA_RX:
 738                 switch (config->dv_xmeta_en) {
 739                 case MLX5_XMETA_MODE_LEGACY:
 740                         return REG_B;
 741                 case MLX5_XMETA_MODE_META16:
 742                         return REG_C_0;
 743                 case MLX5_XMETA_MODE_META32:
 744                         return REG_C_1;
 745                 }
 746                 break;
 747         case MLX5_METADATA_TX:
 748                 return REG_A;
 749         case MLX5_METADATA_FDB:
 750                 switch (config->dv_xmeta_en) {
 751                 case MLX5_XMETA_MODE_LEGACY:
 752                         return REG_NON;
 753                 case MLX5_XMETA_MODE_META16:
 754                         return REG_C_0;
 755                 case MLX5_XMETA_MODE_META32:
 756                         return REG_C_1;
 757                 }
 758                 break;
 759         case MLX5_FLOW_MARK:
 760                 switch (config->dv_xmeta_en) {
 761                 case MLX5_XMETA_MODE_LEGACY:
 762                         return REG_NON;
 763                 case MLX5_XMETA_MODE_META16:
 764                         return REG_C_1;
 765                 case MLX5_XMETA_MODE_META32:
 766                         return REG_C_0;
 767                 }
 768                 break;
 769         case MLX5_MTR_SFX:
 770                 /*
 771                  * If meter color and flow match share one register, flow match
 772                  * should use the meter color register for match.
 773                  */
 774                 if (priv->mtr_reg_share)
 775                         return priv->mtr_color_reg;
 776                 else
 777                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 778                                REG_C_3;
 779         case MLX5_MTR_COLOR:
 780         case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */
 781                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 782                 return priv->mtr_color_reg;
 783         case MLX5_COPY_MARK:
 784                 /*
 785                  * Metadata COPY_MARK register using is in meter suffix sub
 786                  * flow while with meter. It's safe to share the same register.
 787                  */
 788                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 789         case MLX5_APP_TAG:
 790                 /*
 791                  * If meter is enable, it will engage the register for color
 792                  * match and flow match. If meter color match is not using the
 793                  * REG_C_2, need to skip the REG_C_x be used by meter color
 794                  * match.
 795                  * If meter is disable, free to use all available registers.
 796                  */
 797                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 798                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 799                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 800                 if (id > (REG_C_7 - start_reg))
 801                         return rte_flow_error_set(error, EINVAL,
 802                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 803                                                   NULL, "invalid tag id");
 804                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 805                         return rte_flow_error_set(error, ENOTSUP,
 806                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 807                                                   NULL, "unsupported tag id");
 808                 /*
 809                  * This case means meter is using the REG_C_x great than 2.
 810                  * Take care not to conflict with meter color REG_C_x.
 811                  * If the available index REG_C_y >= REG_C_x, skip the
 812                  * color register.
 813                  */
 814                 if (skip_mtr_reg && config->flow_mreg_c
 815                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 816                         if (id >= (REG_C_7 - start_reg))
 817                                 return rte_flow_error_set(error, EINVAL,
 818                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 819                                                         NULL, "invalid tag id");
 820                         if (config->flow_mreg_c
 821                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 822                                 return config->flow_mreg_c
 823                                                [id + 1 + start_reg - REG_C_0];
 824                         return rte_flow_error_set(error, ENOTSUP,
 825                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 826                                                   NULL, "unsupported tag id");
 827                 }
 828                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 829         }
 830         MLX5_ASSERT(false);
 831         return rte_flow_error_set(error, EINVAL,
 832                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 833                                   NULL, "invalid feature name");
 834 }
 835
 836 /**
 837  * Check extensive flow metadata register support.
 838  *
 839  * @param dev
 840  *   Pointer to rte_eth_dev structure.
 841  *
 842  * @return
 843  *   True if device supports extensive flow metadata register, otherwise false.
 844  */
 845 bool
 846 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 847 {
 848         struct mlx5_priv *priv = dev->data->dev_private;
 849         struct mlx5_dev_config *config = &priv->config;
 850
 851         /*
 852          * Having available reg_c can be regarded inclusively as supporting
 853          * extensive flow metadata register, which could mean,
 854          * - metadata register copy action by modify header.
 855          * - 16 modify header actions is supported.
 856          * - reg_c's are preserved across different domain (FDB and NIC) on
 857          *   packet loopback by flow lookup miss.
 858          */
 859         return config->flow_mreg_c[2] != REG_NON;
 860 }
 861
 862 /**
 863  * Verify the @p item specifications (spec, last, mask) are compatible with the
 864  * NIC capabilities.
 865  *
 866  * @param[in] item
 867  *   Item specification.
 868  * @param[in] mask
 869  *   @p item->mask or flow default bit-masks.
 870  * @param[in] nic_mask
 871  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 872  * @param[in] size
 873  *   Bit-masks size in bytes.
 874  * @param[in] range_accepted
 875  *   True if range of values is accepted for specific fields, false otherwise.
 876  * @param[out] error
 877  *   Pointer to error structure.
 878  *
 879  * @return
 880  *   0 on success, a negative errno value otherwise and rte_errno is set.
 881  */
 882 int
 883 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 884                           const uint8_t *mask,
 885                           const uint8_t *nic_mask,
 886                           unsigned int size,
 887                           bool range_accepted,
 888                           struct rte_flow_error *error)
 889 {
 890         unsigned int i;
 891
 892         MLX5_ASSERT(nic_mask);
 893         for (i = 0; i < size; ++i)
 894                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 895                         return rte_flow_error_set(error, ENOTSUP,
 896                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 897                                                   item,
 898                                                   "mask enables non supported"
 899                                                   " bits");
 900         if (!item->spec && (item->mask || item->last))
 901                 return rte_flow_error_set(error, EINVAL,
 902                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 903                                           "mask/last without a spec is not"
 904                                           " supported");
 905         if (item->spec && item->last && !range_accepted) {
 906                 uint8_t spec[size];
 907                 uint8_t last[size];
 908                 unsigned int i;
 909                 int ret;
 910
 911                 for (i = 0; i < size; ++i) {
 912                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 913                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 914                 }
 915                 ret = memcmp(spec, last, size);
 916                 if (ret != 0)
 917                         return rte_flow_error_set(error, EINVAL,
 918                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 919                                                   item,
 920                                                   "range is not valid");
 921         }
 922         return 0;
 923 }
 924
 925 /**
 926  * Adjust the hash fields according to the @p flow information.
 927  *
 928  * @param[in] dev_flow.
 929  *   Pointer to the mlx5_flow.
 930  * @param[in] tunnel
 931  *   1 when the hash field is for a tunnel item.
 932  * @param[in] layer_types
 933  *   ETH_RSS_* types.
 934  * @param[in] hash_fields
 935  *   Item hash fields.
 936  *
 937  * @return
 938  *   The hash fields that should be used.
 939  */
 940 uint64_t
 941 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 942                             int tunnel __rte_unused, uint64_t layer_types,
 943                             uint64_t hash_fields)
 944 {
 945 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 946         int rss_request_inner = rss_desc->level >= 2;
 947
 948         /* Check RSS hash level for tunnel. */
 949         if (tunnel && rss_request_inner)
 950                 hash_fields |= IBV_RX_HASH_INNER;
 951         else if (tunnel || rss_request_inner)
 952                 return 0;
 953 #endif
 954         /* Check if requested layer matches RSS hash fields. */
 955         if (!(rss_desc->types & layer_types))
 956                 return 0;
 957         return hash_fields;
 958 }
 959
 960 /**
 961  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 962  * if several tunnel rules are used on this queue, the tunnel ptype will be
 963  * cleared.
 964  *
 965  * @param rxq_ctrl
 966  *   Rx queue to update.
 967  */
 968 static void
 969 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 970 {
 971         unsigned int i;
 972         uint32_t tunnel_ptype = 0;
 973
 974         /* Look up for the ptype to use. */
 975         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 976                 if (!rxq_ctrl->flow_tunnels_n[i])
 977                         continue;
 978                 if (!tunnel_ptype) {
 979                         tunnel_ptype = tunnels_info[i].ptype;
 980                 } else {
 981                         tunnel_ptype = 0;
 982                         break;
 983                 }
 984         }
 985         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 986 }
 987
 988 /**
 989  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 990  * flow.
 991  *
 992  * @param[in] dev
 993  *   Pointer to the Ethernet device structure.
 994  * @param[in] dev_handle
 995  *   Pointer to device flow handle structure.
 996  */
 997 static void
 998 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
 999                        struct mlx5_flow_handle *dev_handle)
1000 {
1001         struct mlx5_priv *priv = dev->data->dev_private;
1002         const int mark = dev_handle->mark;
1003         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1004         struct mlx5_hrxq *hrxq;
1005         unsigned int i;
1006
1007         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1008                 return;
1009         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1010                               dev_handle->rix_hrxq);
1011         if (!hrxq)
1012                 return;
1013         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1014                 int idx = hrxq->ind_table->queues[i];
1015                 struct mlx5_rxq_ctrl *rxq_ctrl =
1016                         container_of((*priv->rxqs)[idx],
1017                                      struct mlx5_rxq_ctrl, rxq);
1018
1019                 /*
1020                  * To support metadata register copy on Tx loopback,
1021                  * this must be always enabled (metadata may arive
1022                  * from other port - not from local flows only.
1023                  */
1024                 if (priv->config.dv_flow_en &&
1025                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1026                     mlx5_flow_ext_mreg_supported(dev)) {
1027                         rxq_ctrl->rxq.mark = 1;
1028                         rxq_ctrl->flow_mark_n = 1;
1029                 } else if (mark) {
1030                         rxq_ctrl->rxq.mark = 1;
1031                         rxq_ctrl->flow_mark_n++;
1032                 }
1033                 if (tunnel) {
1034                         unsigned int j;
1035
1036                         /* Increase the counter matching the flow. */
1037                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1038                                 if ((tunnels_info[j].tunnel &
1039                                      dev_handle->layers) ==
1040                                     tunnels_info[j].tunnel) {
1041                                         rxq_ctrl->flow_tunnels_n[j]++;
1042                                         break;
1043                                 }
1044                         }
1045                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1046                 }
1047         }
1048 }
1049
1050 /**
1051  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1052  *
1053  * @param[in] dev
1054  *   Pointer to the Ethernet device structure.
1055  * @param[in] flow
1056  *   Pointer to flow structure.
1057  */
1058 static void
1059 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1060 {
1061         struct mlx5_priv *priv = dev->data->dev_private;
1062         uint32_t handle_idx;
1063         struct mlx5_flow_handle *dev_handle;
1064
1065         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1066                        handle_idx, dev_handle, next)
1067                 flow_drv_rxq_flags_set(dev, dev_handle);
1068 }
1069
1070 /**
1071  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1072  * device flow if no other flow uses it with the same kind of request.
1073  *
1074  * @param dev
1075  *   Pointer to Ethernet device.
1076  * @param[in] dev_handle
1077  *   Pointer to the device flow handle structure.
1078  */
1079 static void
1080 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1081                         struct mlx5_flow_handle *dev_handle)
1082 {
1083         struct mlx5_priv *priv = dev->data->dev_private;
1084         const int mark = dev_handle->mark;
1085         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1086         struct mlx5_hrxq *hrxq;
1087         unsigned int i;
1088
1089         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1090                 return;
1091         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1092                               dev_handle->rix_hrxq);
1093         if (!hrxq)
1094                 return;
1095         MLX5_ASSERT(dev->data->dev_started);
1096         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1097                 int idx = hrxq->ind_table->queues[i];
1098                 struct mlx5_rxq_ctrl *rxq_ctrl =
1099                         container_of((*priv->rxqs)[idx],
1100                                      struct mlx5_rxq_ctrl, rxq);
1101
1102                 if (priv->config.dv_flow_en &&
1103                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1104                     mlx5_flow_ext_mreg_supported(dev)) {
1105                         rxq_ctrl->rxq.mark = 1;
1106                         rxq_ctrl->flow_mark_n = 1;
1107                 } else if (mark) {
1108                         rxq_ctrl->flow_mark_n--;
1109                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1110                 }
1111                 if (tunnel) {
1112                         unsigned int j;
1113
1114                         /* Decrease the counter matching the flow. */
1115                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1116                                 if ((tunnels_info[j].tunnel &
1117                                      dev_handle->layers) ==
1118                                     tunnels_info[j].tunnel) {
1119                                         rxq_ctrl->flow_tunnels_n[j]--;
1120                                         break;
1121                                 }
1122                         }
1123                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1124                 }
1125         }
1126 }
1127
1128 /**
1129  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1130  * @p flow if no other flow uses it with the same kind of request.
1131  *
1132  * @param dev
1133  *   Pointer to Ethernet device.
1134  * @param[in] flow
1135  *   Pointer to the flow.
1136  */
1137 static void
1138 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1139 {
1140         struct mlx5_priv *priv = dev->data->dev_private;
1141         uint32_t handle_idx;
1142         struct mlx5_flow_handle *dev_handle;
1143
1144         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1145                        handle_idx, dev_handle, next)
1146                 flow_drv_rxq_flags_trim(dev, dev_handle);
1147 }
1148
1149 /**
1150  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1151  *
1152  * @param dev
1153  *   Pointer to Ethernet device.
1154  */
1155 static void
1156 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1157 {
1158         struct mlx5_priv *priv = dev->data->dev_private;
1159         unsigned int i;
1160
1161         for (i = 0; i != priv->rxqs_n; ++i) {
1162                 struct mlx5_rxq_ctrl *rxq_ctrl;
1163                 unsigned int j;
1164
1165                 if (!(*priv->rxqs)[i])
1166                         continue;
1167                 rxq_ctrl = container_of((*priv->rxqs)[i],
1168                                         struct mlx5_rxq_ctrl, rxq);
1169                 rxq_ctrl->flow_mark_n = 0;
1170                 rxq_ctrl->rxq.mark = 0;
1171                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1172                         rxq_ctrl->flow_tunnels_n[j] = 0;
1173                 rxq_ctrl->rxq.tunnel = 0;
1174         }
1175 }
1176
1177 /**
1178  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1179  *
1180  * @param[in] dev
1181  *   Pointer to the Ethernet device structure.
1182  */
1183 void
1184 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1185 {
1186         struct mlx5_priv *priv = dev->data->dev_private;
1187         struct mlx5_rxq_data *data;
1188         unsigned int i;
1189
1190         for (i = 0; i != priv->rxqs_n; ++i) {
1191                 if (!(*priv->rxqs)[i])
1192                         continue;
1193                 data = (*priv->rxqs)[i];
1194                 if (!rte_flow_dynf_metadata_avail()) {
1195                         data->dynf_meta = 0;
1196                         data->flow_meta_mask = 0;
1197                         data->flow_meta_offset = -1;
1198                 } else {
1199                         data->dynf_meta = 1;
1200                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1201                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1202                 }
1203         }
1204 }
1205
1206 /*
1207  * return a pointer to the desired action in the list of actions.
1208  *
1209  * @param[in] actions
1210  *   The list of actions to search the action in.
1211  * @param[in] action
1212  *   The action to find.
1213  *
1214  * @return
1215  *   Pointer to the action in the list, if found. NULL otherwise.
1216  */
1217 const struct rte_flow_action *
1218 mlx5_flow_find_action(const struct rte_flow_action *actions,
1219                       enum rte_flow_action_type action)
1220 {
1221         if (actions == NULL)
1222                 return NULL;
1223         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1224                 if (actions->type == action)
1225                         return actions;
1226         return NULL;
1227 }
1228
1229 /*
1230  * Validate the flag action.
1231  *
1232  * @param[in] action_flags
1233  *   Bit-fields that holds the actions detected until now.
1234  * @param[in] attr
1235  *   Attributes of flow that includes this action.
1236  * @param[out] error
1237  *   Pointer to error structure.
1238  *
1239  * @return
1240  *   0 on success, a negative errno value otherwise and rte_errno is set.
1241  */
1242 int
1243 mlx5_flow_validate_action_flag(uint64_t action_flags,
1244                                const struct rte_flow_attr *attr,
1245                                struct rte_flow_error *error)
1246 {
1247         if (action_flags & MLX5_FLOW_ACTION_MARK)
1248                 return rte_flow_error_set(error, EINVAL,
1249                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1250                                           "can't mark and flag in same flow");
1251         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1252                 return rte_flow_error_set(error, EINVAL,
1253                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1254                                           "can't have 2 flag"
1255                                           " actions in same flow");
1256         if (attr->egress)
1257                 return rte_flow_error_set(error, ENOTSUP,
1258                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1259                                           "flag action not supported for "
1260                                           "egress");
1261         return 0;
1262 }
1263
1264 /*
1265  * Validate the mark action.
1266  *
1267  * @param[in] action
1268  *   Pointer to the queue action.
1269  * @param[in] action_flags
1270  *   Bit-fields that holds the actions detected until now.
1271  * @param[in] attr
1272  *   Attributes of flow that includes this action.
1273  * @param[out] error
1274  *   Pointer to error structure.
1275  *
1276  * @return
1277  *   0 on success, a negative errno value otherwise and rte_errno is set.
1278  */
1279 int
1280 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1281                                uint64_t action_flags,
1282                                const struct rte_flow_attr *attr,
1283                                struct rte_flow_error *error)
1284 {
1285         const struct rte_flow_action_mark *mark = action->conf;
1286
1287         if (!mark)
1288                 return rte_flow_error_set(error, EINVAL,
1289                                           RTE_FLOW_ERROR_TYPE_ACTION,
1290                                           action,
1291                                           "configuration cannot be null");
1292         if (mark->id >= MLX5_FLOW_MARK_MAX)
1293                 return rte_flow_error_set(error, EINVAL,
1294                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1295                                           &mark->id,
1296                                           "mark id must in 0 <= id < "
1297                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1298         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1299                 return rte_flow_error_set(error, EINVAL,
1300                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1301                                           "can't flag and mark in same flow");
1302         if (action_flags & MLX5_FLOW_ACTION_MARK)
1303                 return rte_flow_error_set(error, EINVAL,
1304                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1305                                           "can't have 2 mark actions in same"
1306                                           " flow");
1307         if (attr->egress)
1308                 return rte_flow_error_set(error, ENOTSUP,
1309                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1310                                           "mark action not supported for "
1311                                           "egress");
1312         return 0;
1313 }
1314
1315 /*
1316  * Validate the drop action.
1317  *
1318  * @param[in] action_flags
1319  *   Bit-fields that holds the actions detected until now.
1320  * @param[in] attr
1321  *   Attributes of flow that includes this action.
1322  * @param[out] error
1323  *   Pointer to error structure.
1324  *
1325  * @return
1326  *   0 on success, a negative errno value otherwise and rte_errno is set.
1327  */
1328 int
1329 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1330                                const struct rte_flow_attr *attr,
1331                                struct rte_flow_error *error)
1332 {
1333         if (attr->egress)
1334                 return rte_flow_error_set(error, ENOTSUP,
1335                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1336                                           "drop action not supported for "
1337                                           "egress");
1338         return 0;
1339 }
1340
1341 /*
1342  * Validate the queue action.
1343  *
1344  * @param[in] action
1345  *   Pointer to the queue action.
1346  * @param[in] action_flags
1347  *   Bit-fields that holds the actions detected until now.
1348  * @param[in] dev
1349  *   Pointer to the Ethernet device structure.
1350  * @param[in] attr
1351  *   Attributes of flow that includes this action.
1352  * @param[out] error
1353  *   Pointer to error structure.
1354  *
1355  * @return
1356  *   0 on success, a negative errno value otherwise and rte_errno is set.
1357  */
1358 int
1359 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1360                                 uint64_t action_flags,
1361                                 struct rte_eth_dev *dev,
1362                                 const struct rte_flow_attr *attr,
1363                                 struct rte_flow_error *error)
1364 {
1365         struct mlx5_priv *priv = dev->data->dev_private;
1366         const struct rte_flow_action_queue *queue = action->conf;
1367
1368         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1369                 return rte_flow_error_set(error, EINVAL,
1370                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1371                                           "can't have 2 fate actions in"
1372                                           " same flow");
1373         if (!priv->rxqs_n)
1374                 return rte_flow_error_set(error, EINVAL,
1375                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1376                                           NULL, "No Rx queues configured");
1377         if (queue->index >= priv->rxqs_n)
1378                 return rte_flow_error_set(error, EINVAL,
1379                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1380                                           &queue->index,
1381                                           "queue index out of range");
1382         if (!(*priv->rxqs)[queue->index])
1383                 return rte_flow_error_set(error, EINVAL,
1384                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1385                                           &queue->index,
1386                                           "queue is not configured");
1387         if (attr->egress)
1388                 return rte_flow_error_set(error, ENOTSUP,
1389                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1390                                           "queue action not supported for "
1391                                           "egress");
1392         return 0;
1393 }
1394
1395 /*
1396  * Validate the rss action.
1397  *
1398  * @param[in] dev
1399  *   Pointer to the Ethernet device structure.
1400  * @param[in] action
1401  *   Pointer to the queue action.
1402  * @param[out] error
1403  *   Pointer to error structure.
1404  *
1405  * @return
1406  *   0 on success, a negative errno value otherwise and rte_errno is set.
1407  */
1408 int
1409 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1410                          const struct rte_flow_action *action,
1411                          struct rte_flow_error *error)
1412 {
1413         struct mlx5_priv *priv = dev->data->dev_private;
1414         const struct rte_flow_action_rss *rss = action->conf;
1415         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1416         unsigned int i;
1417
1418         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1419             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1420                 return rte_flow_error_set(error, ENOTSUP,
1421                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1422                                           &rss->func,
1423                                           "RSS hash function not supported");
1424 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1425         if (rss->level > 2)
1426 #else
1427         if (rss->level > 1)
1428 #endif
1429                 return rte_flow_error_set(error, ENOTSUP,
1430                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1431                                           &rss->level,
1432                                           "tunnel RSS is not supported");
1433         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1434         if (rss->key_len == 0 && rss->key != NULL)
1435                 return rte_flow_error_set(error, ENOTSUP,
1436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1437                                           &rss->key_len,
1438                                           "RSS hash key length 0");
1439         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1440                 return rte_flow_error_set(error, ENOTSUP,
1441                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1442                                           &rss->key_len,
1443                                           "RSS hash key too small");
1444         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1445                 return rte_flow_error_set(error, ENOTSUP,
1446                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1447                                           &rss->key_len,
1448                                           "RSS hash key too large");
1449         if (rss->queue_num > priv->config.ind_table_max_size)
1450                 return rte_flow_error_set(error, ENOTSUP,
1451                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1452                                           &rss->queue_num,
1453                                           "number of queues too large");
1454         if (rss->types & MLX5_RSS_HF_MASK)
1455                 return rte_flow_error_set(error, ENOTSUP,
1456                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1457                                           &rss->types,
1458                                           "some RSS protocols are not"
1459                                           " supported");
1460         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1461             !(rss->types & ETH_RSS_IP))
1462                 return rte_flow_error_set(error, EINVAL,
1463                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1464                                           "L3 partial RSS requested but L3 RSS"
1465                                           " type not specified");
1466         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1467             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1470                                           "L4 partial RSS requested but L4 RSS"
1471                                           " type not specified");
1472         if (!priv->rxqs_n)
1473                 return rte_flow_error_set(error, EINVAL,
1474                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1475                                           NULL, "No Rx queues configured");
1476         if (!rss->queue_num)
1477                 return rte_flow_error_set(error, EINVAL,
1478                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1479                                           NULL, "No queues configured");
1480         for (i = 0; i != rss->queue_num; ++i) {
1481                 struct mlx5_rxq_ctrl *rxq_ctrl;
1482
1483                 if (rss->queue[i] >= priv->rxqs_n)
1484                         return rte_flow_error_set
1485                                 (error, EINVAL,
1486                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1487                                  &rss->queue[i], "queue index out of range");
1488                 if (!(*priv->rxqs)[rss->queue[i]])
1489                         return rte_flow_error_set
1490                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1491                                  &rss->queue[i], "queue is not configured");
1492                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1493                                         struct mlx5_rxq_ctrl, rxq);
1494                 if (i == 0)
1495                         rxq_type = rxq_ctrl->type;
1496                 if (rxq_type != rxq_ctrl->type)
1497                         return rte_flow_error_set
1498                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1499                                  &rss->queue[i],
1500                                  "combining hairpin and regular RSS queues is not supported");
1501         }
1502         return 0;
1503 }
1504
1505 /*
1506  * Validate the rss action.
1507  *
1508  * @param[in] action
1509  *   Pointer to the queue action.
1510  * @param[in] action_flags
1511  *   Bit-fields that holds the actions detected until now.
1512  * @param[in] dev
1513  *   Pointer to the Ethernet device structure.
1514  * @param[in] attr
1515  *   Attributes of flow that includes this action.
1516  * @param[in] item_flags
1517  *   Items that were detected.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 int
1525 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1526                               uint64_t action_flags,
1527                               struct rte_eth_dev *dev,
1528                               const struct rte_flow_attr *attr,
1529                               uint64_t item_flags,
1530                               struct rte_flow_error *error)
1531 {
1532         const struct rte_flow_action_rss *rss = action->conf;
1533         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1534         int ret;
1535
1536         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1539                                           "can't have 2 fate actions"
1540                                           " in same flow");
1541         ret = mlx5_validate_action_rss(dev, action, error);
1542         if (ret)
1543                 return ret;
1544         if (attr->egress)
1545                 return rte_flow_error_set(error, ENOTSUP,
1546                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1547                                           "rss action not supported for "
1548                                           "egress");
1549         if (rss->level > 1 && !tunnel)
1550                 return rte_flow_error_set(error, EINVAL,
1551                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1552                                           "inner RSS is not supported for "
1553                                           "non-tunnel flows");
1554         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1555             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1556                 return rte_flow_error_set(error, EINVAL,
1557                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1558                                           "RSS on eCPRI is not supported now");
1559         }
1560         return 0;
1561 }
1562
1563 /*
1564  * Validate the default miss action.
1565  *
1566  * @param[in] action_flags
1567  *   Bit-fields that holds the actions detected until now.
1568  * @param[out] error
1569  *   Pointer to error structure.
1570  *
1571  * @return
1572  *   0 on success, a negative errno value otherwise and rte_errno is set.
1573  */
1574 int
1575 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1576                                 const struct rte_flow_attr *attr,
1577                                 struct rte_flow_error *error)
1578 {
1579         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1580                 return rte_flow_error_set(error, EINVAL,
1581                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1582                                           "can't have 2 fate actions in"
1583                                           " same flow");
1584         if (attr->egress)
1585                 return rte_flow_error_set(error, ENOTSUP,
1586                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1587                                           "default miss action not supported "
1588                                           "for egress");
1589         if (attr->group)
1590                 return rte_flow_error_set(error, ENOTSUP,
1591                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1592                                           "only group 0 is supported");
1593         if (attr->transfer)
1594                 return rte_flow_error_set(error, ENOTSUP,
1595                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1596                                           NULL, "transfer is not supported");
1597         return 0;
1598 }
1599
1600 /*
1601  * Validate the count action.
1602  *
1603  * @param[in] dev
1604  *   Pointer to the Ethernet device structure.
1605  * @param[in] attr
1606  *   Attributes of flow that includes this action.
1607  * @param[out] error
1608  *   Pointer to error structure.
1609  *
1610  * @return
1611  *   0 on success, a negative errno value otherwise and rte_errno is set.
1612  */
1613 int
1614 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1615                                 const struct rte_flow_attr *attr,
1616                                 struct rte_flow_error *error)
1617 {
1618         if (attr->egress)
1619                 return rte_flow_error_set(error, ENOTSUP,
1620                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1621                                           "count action not supported for "
1622                                           "egress");
1623         return 0;
1624 }
1625
1626 /**
1627  * Verify the @p attributes will be correctly understood by the NIC and store
1628  * them in the @p flow if everything is correct.
1629  *
1630  * @param[in] dev
1631  *   Pointer to the Ethernet device structure.
1632  * @param[in] attributes
1633  *   Pointer to flow attributes
1634  * @param[out] error
1635  *   Pointer to error structure.
1636  *
1637  * @return
1638  *   0 on success, a negative errno value otherwise and rte_errno is set.
1639  */
1640 int
1641 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1642                               const struct rte_flow_attr *attributes,
1643                               struct rte_flow_error *error)
1644 {
1645         struct mlx5_priv *priv = dev->data->dev_private;
1646         uint32_t priority_max = priv->config.flow_prio - 1;
1647
1648         if (attributes->group)
1649                 return rte_flow_error_set(error, ENOTSUP,
1650                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1651                                           NULL, "groups is not supported");
1652         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1653             attributes->priority >= priority_max)
1654                 return rte_flow_error_set(error, ENOTSUP,
1655                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1656                                           NULL, "priority out of range");
1657         if (attributes->egress)
1658                 return rte_flow_error_set(error, ENOTSUP,
1659                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1660                                           "egress is not supported");
1661         if (attributes->transfer && !priv->config.dv_esw_en)
1662                 return rte_flow_error_set(error, ENOTSUP,
1663                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1664                                           NULL, "transfer is not supported");
1665         if (!attributes->ingress)
1666                 return rte_flow_error_set(error, EINVAL,
1667                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1668                                           NULL,
1669                                           "ingress attribute is mandatory");
1670         return 0;
1671 }
1672
1673 /**
1674  * Validate ICMP6 item.
1675  *
1676  * @param[in] item
1677  *   Item specification.
1678  * @param[in] item_flags
1679  *   Bit-fields that holds the items detected until now.
1680  * @param[in] ext_vlan_sup
1681  *   Whether extended VLAN features are supported or not.
1682  * @param[out] error
1683  *   Pointer to error structure.
1684  *
1685  * @return
1686  *   0 on success, a negative errno value otherwise and rte_errno is set.
1687  */
1688 int
1689 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1690                                uint64_t item_flags,
1691                                uint8_t target_protocol,
1692                                struct rte_flow_error *error)
1693 {
1694         const struct rte_flow_item_icmp6 *mask = item->mask;
1695         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1696         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1697                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1698         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1699                                       MLX5_FLOW_LAYER_OUTER_L4;
1700         int ret;
1701
1702         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1703                 return rte_flow_error_set(error, EINVAL,
1704                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1705                                           "protocol filtering not compatible"
1706                                           " with ICMP6 layer");
1707         if (!(item_flags & l3m))
1708                 return rte_flow_error_set(error, EINVAL,
1709                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1710                                           "IPv6 is mandatory to filter on"
1711                                           " ICMP6");
1712         if (item_flags & l4m)
1713                 return rte_flow_error_set(error, EINVAL,
1714                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1715                                           "multiple L4 layers not supported");
1716         if (!mask)
1717                 mask = &rte_flow_item_icmp6_mask;
1718         ret = mlx5_flow_item_acceptable
1719                 (item, (const uint8_t *)mask,
1720                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1721                  sizeof(struct rte_flow_item_icmp6),
1722                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1723         if (ret < 0)
1724                 return ret;
1725         return 0;
1726 }
1727
1728 /**
1729  * Validate ICMP item.
1730  *
1731  * @param[in] item
1732  *   Item specification.
1733  * @param[in] item_flags
1734  *   Bit-fields that holds the items detected until now.
1735  * @param[out] error
1736  *   Pointer to error structure.
1737  *
1738  * @return
1739  *   0 on success, a negative errno value otherwise and rte_errno is set.
1740  */
1741 int
1742 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1743                              uint64_t item_flags,
1744                              uint8_t target_protocol,
1745                              struct rte_flow_error *error)
1746 {
1747         const struct rte_flow_item_icmp *mask = item->mask;
1748         const struct rte_flow_item_icmp nic_mask = {
1749                 .hdr.icmp_type = 0xff,
1750                 .hdr.icmp_code = 0xff,
1751                 .hdr.icmp_ident = RTE_BE16(0xffff),
1752                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1753         };
1754         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1755         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1756                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1757         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1758                                       MLX5_FLOW_LAYER_OUTER_L4;
1759         int ret;
1760
1761         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1762                 return rte_flow_error_set(error, EINVAL,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1764                                           "protocol filtering not compatible"
1765                                           " with ICMP layer");
1766         if (!(item_flags & l3m))
1767                 return rte_flow_error_set(error, EINVAL,
1768                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1769                                           "IPv4 is mandatory to filter"
1770                                           " on ICMP");
1771         if (item_flags & l4m)
1772                 return rte_flow_error_set(error, EINVAL,
1773                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1774                                           "multiple L4 layers not supported");
1775         if (!mask)
1776                 mask = &nic_mask;
1777         ret = mlx5_flow_item_acceptable
1778                 (item, (const uint8_t *)mask,
1779                  (const uint8_t *)&nic_mask,
1780                  sizeof(struct rte_flow_item_icmp),
1781                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1782         if (ret < 0)
1783                 return ret;
1784         return 0;
1785 }
1786
1787 /**
1788  * Validate Ethernet item.
1789  *
1790  * @param[in] item
1791  *   Item specification.
1792  * @param[in] item_flags
1793  *   Bit-fields that holds the items detected until now.
1794  * @param[out] error
1795  *   Pointer to error structure.
1796  *
1797  * @return
1798  *   0 on success, a negative errno value otherwise and rte_errno is set.
1799  */
1800 int
1801 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1802                             uint64_t item_flags, bool ext_vlan_sup,
1803                             struct rte_flow_error *error)
1804 {
1805         const struct rte_flow_item_eth *mask = item->mask;
1806         const struct rte_flow_item_eth nic_mask = {
1807                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1808                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1809                 .type = RTE_BE16(0xffff),
1810                 .has_vlan = ext_vlan_sup ? 1 : 0,
1811         };
1812         int ret;
1813         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1814         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1815                                        MLX5_FLOW_LAYER_OUTER_L2;
1816
1817         if (item_flags & ethm)
1818                 return rte_flow_error_set(error, ENOTSUP,
1819                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1820                                           "multiple L2 layers not supported");
1821         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1822             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1823                 return rte_flow_error_set(error, EINVAL,
1824                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1825                                           "L2 layer should not follow "
1826                                           "L3 layers");
1827         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1828             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1829                 return rte_flow_error_set(error, EINVAL,
1830                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1831                                           "L2 layer should not follow VLAN");
1832         if (!mask)
1833                 mask = &rte_flow_item_eth_mask;
1834         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1835                                         (const uint8_t *)&nic_mask,
1836                                         sizeof(struct rte_flow_item_eth),
1837                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1838         return ret;
1839 }
1840
1841 /**
1842  * Validate VLAN item.
1843  *
1844  * @param[in] item
1845  *   Item specification.
1846  * @param[in] item_flags
1847  *   Bit-fields that holds the items detected until now.
1848  * @param[in] dev
1849  *   Ethernet device flow is being created on.
1850  * @param[out] error
1851  *   Pointer to error structure.
1852  *
1853  * @return
1854  *   0 on success, a negative errno value otherwise and rte_errno is set.
1855  */
1856 int
1857 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1858                              uint64_t item_flags,
1859                              struct rte_eth_dev *dev,
1860                              struct rte_flow_error *error)
1861 {
1862         const struct rte_flow_item_vlan *spec = item->spec;
1863         const struct rte_flow_item_vlan *mask = item->mask;
1864         const struct rte_flow_item_vlan nic_mask = {
1865                 .tci = RTE_BE16(UINT16_MAX),
1866                 .inner_type = RTE_BE16(UINT16_MAX),
1867         };
1868         uint16_t vlan_tag = 0;
1869         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1870         int ret;
1871         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1872                                         MLX5_FLOW_LAYER_INNER_L4) :
1873                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1874                                         MLX5_FLOW_LAYER_OUTER_L4);
1875         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1876                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1877
1878         if (item_flags & vlanm)
1879                 return rte_flow_error_set(error, EINVAL,
1880                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1881                                           "multiple VLAN layers not supported");
1882         else if ((item_flags & l34m) != 0)
1883                 return rte_flow_error_set(error, EINVAL,
1884                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1885                                           "VLAN cannot follow L3/L4 layer");
1886         if (!mask)
1887                 mask = &rte_flow_item_vlan_mask;
1888         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1889                                         (const uint8_t *)&nic_mask,
1890                                         sizeof(struct rte_flow_item_vlan),
1891                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1892         if (ret)
1893                 return ret;
1894         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1895                 struct mlx5_priv *priv = dev->data->dev_private;
1896
1897                 if (priv->vmwa_context) {
1898                         /*
1899                          * Non-NULL context means we have a virtual machine
1900                          * and SR-IOV enabled, we have to create VLAN interface
1901                          * to make hypervisor to setup E-Switch vport
1902                          * context correctly. We avoid creating the multiple
1903                          * VLAN interfaces, so we cannot support VLAN tag mask.
1904                          */
1905                         return rte_flow_error_set(error, EINVAL,
1906                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1907                                                   item,
1908                                                   "VLAN tag mask is not"
1909                                                   " supported in virtual"
1910                                                   " environment");
1911                 }
1912         }
1913         if (spec) {
1914                 vlan_tag = spec->tci;
1915                 vlan_tag &= mask->tci;
1916         }
1917         /*
1918          * From verbs perspective an empty VLAN is equivalent
1919          * to a packet without VLAN layer.
1920          */
1921         if (!vlan_tag)
1922                 return rte_flow_error_set(error, EINVAL,
1923                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1924                                           item->spec,
1925                                           "VLAN cannot be empty");
1926         return 0;
1927 }
1928
1929 /**
1930  * Validate IPV4 item.
1931  *
1932  * @param[in] item
1933  *   Item specification.
1934  * @param[in] item_flags
1935  *   Bit-fields that holds the items detected until now.
1936  * @param[in] last_item
1937  *   Previous validated item in the pattern items.
1938  * @param[in] ether_type
1939  *   Type in the ethernet layer header (including dot1q).
1940  * @param[in] acc_mask
1941  *   Acceptable mask, if NULL default internal default mask
1942  *   will be used to check whether item fields are supported.
1943  * @param[in] range_accepted
1944  *   True if range of values is accepted for specific fields, false otherwise.
1945  * @param[out] error
1946  *   Pointer to error structure.
1947  *
1948  * @return
1949  *   0 on success, a negative errno value otherwise and rte_errno is set.
1950  */
1951 int
1952 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1953                              uint64_t item_flags,
1954                              uint64_t last_item,
1955                              uint16_t ether_type,
1956                              const struct rte_flow_item_ipv4 *acc_mask,
1957                              bool range_accepted,
1958                              struct rte_flow_error *error)
1959 {
1960         const struct rte_flow_item_ipv4 *mask = item->mask;
1961         const struct rte_flow_item_ipv4 *spec = item->spec;
1962         const struct rte_flow_item_ipv4 nic_mask = {
1963                 .hdr = {
1964                         .src_addr = RTE_BE32(0xffffffff),
1965                         .dst_addr = RTE_BE32(0xffffffff),
1966                         .type_of_service = 0xff,
1967                         .next_proto_id = 0xff,
1968                 },
1969         };
1970         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1971         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1972                                       MLX5_FLOW_LAYER_OUTER_L3;
1973         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1974                                       MLX5_FLOW_LAYER_OUTER_L4;
1975         int ret;
1976         uint8_t next_proto = 0xFF;
1977         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1978                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1979                                   MLX5_FLOW_LAYER_INNER_VLAN);
1980
1981         if ((last_item & l2_vlan) && ether_type &&
1982             ether_type != RTE_ETHER_TYPE_IPV4)
1983                 return rte_flow_error_set(error, EINVAL,
1984                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1985                                           "IPv4 cannot follow L2/VLAN layer "
1986                                           "which ether type is not IPv4");
1987         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1988                 if (mask && spec)
1989                         next_proto = mask->hdr.next_proto_id &
1990                                      spec->hdr.next_proto_id;
1991                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1992                         return rte_flow_error_set(error, EINVAL,
1993                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1994                                                   item,
1995                                                   "multiple tunnel "
1996                                                   "not supported");
1997         }
1998         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1999                 return rte_flow_error_set(error, EINVAL,
2000                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2001                                           "wrong tunnel type - IPv6 specified "
2002                                           "but IPv4 item provided");
2003         if (item_flags & l3m)
2004                 return rte_flow_error_set(error, ENOTSUP,
2005                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2006                                           "multiple L3 layers not supported");
2007         else if (item_flags & l4m)
2008                 return rte_flow_error_set(error, EINVAL,
2009                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2010                                           "L3 cannot follow an L4 layer.");
2011         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2012                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2013                 return rte_flow_error_set(error, EINVAL,
2014                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2015                                           "L3 cannot follow an NVGRE layer.");
2016         if (!mask)
2017                 mask = &rte_flow_item_ipv4_mask;
2018         else if (mask->hdr.next_proto_id != 0 &&
2019                  mask->hdr.next_proto_id != 0xff)
2020                 return rte_flow_error_set(error, EINVAL,
2021                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2022                                           "partial mask is not supported"
2023                                           " for protocol");
2024         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2025                                         acc_mask ? (const uint8_t *)acc_mask
2026                                                  : (const uint8_t *)&nic_mask,
2027                                         sizeof(struct rte_flow_item_ipv4),
2028                                         range_accepted, error);
2029         if (ret < 0)
2030                 return ret;
2031         return 0;
2032 }
2033
2034 /**
2035  * Validate IPV6 item.
2036  *
2037  * @param[in] item
2038  *   Item specification.
2039  * @param[in] item_flags
2040  *   Bit-fields that holds the items detected until now.
2041  * @param[in] last_item
2042  *   Previous validated item in the pattern items.
2043  * @param[in] ether_type
2044  *   Type in the ethernet layer header (including dot1q).
2045  * @param[in] acc_mask
2046  *   Acceptable mask, if NULL default internal default mask
2047  *   will be used to check whether item fields are supported.
2048  * @param[out] error
2049  *   Pointer to error structure.
2050  *
2051  * @return
2052  *   0 on success, a negative errno value otherwise and rte_errno is set.
2053  */
2054 int
2055 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2056                              uint64_t item_flags,
2057                              uint64_t last_item,
2058                              uint16_t ether_type,
2059                              const struct rte_flow_item_ipv6 *acc_mask,
2060                              struct rte_flow_error *error)
2061 {
2062         const struct rte_flow_item_ipv6 *mask = item->mask;
2063         const struct rte_flow_item_ipv6 *spec = item->spec;
2064         const struct rte_flow_item_ipv6 nic_mask = {
2065                 .hdr = {
2066                         .src_addr =
2067                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2068                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2069                         .dst_addr =
2070                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2071                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2072                         .vtc_flow = RTE_BE32(0xffffffff),
2073                         .proto = 0xff,
2074                 },
2075         };
2076         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2077         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2078                                       MLX5_FLOW_LAYER_OUTER_L3;
2079         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2080                                       MLX5_FLOW_LAYER_OUTER_L4;
2081         int ret;
2082         uint8_t next_proto = 0xFF;
2083         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2084                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2085                                   MLX5_FLOW_LAYER_INNER_VLAN);
2086
2087         if ((last_item & l2_vlan) && ether_type &&
2088             ether_type != RTE_ETHER_TYPE_IPV6)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2091                                           "IPv6 cannot follow L2/VLAN layer "
2092                                           "which ether type is not IPv6");
2093         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2094                 next_proto = spec->hdr.proto;
2095         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2096                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2097                         return rte_flow_error_set(error, EINVAL,
2098                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2099                                                   item,
2100                                                   "multiple tunnel "
2101                                                   "not supported");
2102         }
2103         if (next_proto == IPPROTO_HOPOPTS  ||
2104             next_proto == IPPROTO_ROUTING  ||
2105             next_proto == IPPROTO_FRAGMENT ||
2106             next_proto == IPPROTO_ESP      ||
2107             next_proto == IPPROTO_AH       ||
2108             next_proto == IPPROTO_DSTOPTS)
2109                 return rte_flow_error_set(error, EINVAL,
2110                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2111                                           "IPv6 proto (next header) should "
2112                                           "not be set as extension header");
2113         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2114                 return rte_flow_error_set(error, EINVAL,
2115                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2116                                           "wrong tunnel type - IPv4 specified "
2117                                           "but IPv6 item provided");
2118         if (item_flags & l3m)
2119                 return rte_flow_error_set(error, ENOTSUP,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2121                                           "multiple L3 layers not supported");
2122         else if (item_flags & l4m)
2123                 return rte_flow_error_set(error, EINVAL,
2124                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2125                                           "L3 cannot follow an L4 layer.");
2126         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2127                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2128                 return rte_flow_error_set(error, EINVAL,
2129                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2130                                           "L3 cannot follow an NVGRE layer.");
2131         if (!mask)
2132                 mask = &rte_flow_item_ipv6_mask;
2133         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2134                                         acc_mask ? (const uint8_t *)acc_mask
2135                                                  : (const uint8_t *)&nic_mask,
2136                                         sizeof(struct rte_flow_item_ipv6),
2137                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2138         if (ret < 0)
2139                 return ret;
2140         return 0;
2141 }
2142
2143 /**
2144  * Validate UDP item.
2145  *
2146  * @param[in] item
2147  *   Item specification.
2148  * @param[in] item_flags
2149  *   Bit-fields that holds the items detected until now.
2150  * @param[in] target_protocol
2151  *   The next protocol in the previous item.
2152  * @param[in] flow_mask
2153  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2154  * @param[out] error
2155  *   Pointer to error structure.
2156  *
2157  * @return
2158  *   0 on success, a negative errno value otherwise and rte_errno is set.
2159  */
2160 int
2161 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2162                             uint64_t item_flags,
2163                             uint8_t target_protocol,
2164                             struct rte_flow_error *error)
2165 {
2166         const struct rte_flow_item_udp *mask = item->mask;
2167         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2168         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2169                                       MLX5_FLOW_LAYER_OUTER_L3;
2170         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2171                                       MLX5_FLOW_LAYER_OUTER_L4;
2172         int ret;
2173
2174         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2175                 return rte_flow_error_set(error, EINVAL,
2176                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2177                                           "protocol filtering not compatible"
2178                                           " with UDP layer");
2179         if (!(item_flags & l3m))
2180                 return rte_flow_error_set(error, EINVAL,
2181                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2182                                           "L3 is mandatory to filter on L4");
2183         if (item_flags & l4m)
2184                 return rte_flow_error_set(error, EINVAL,
2185                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2186                                           "multiple L4 layers not supported");
2187         if (!mask)
2188                 mask = &rte_flow_item_udp_mask;
2189         ret = mlx5_flow_item_acceptable
2190                 (item, (const uint8_t *)mask,
2191                  (const uint8_t *)&rte_flow_item_udp_mask,
2192                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2193                  error);
2194         if (ret < 0)
2195                 return ret;
2196         return 0;
2197 }
2198
2199 /**
2200  * Validate TCP item.
2201  *
2202  * @param[in] item
2203  *   Item specification.
2204  * @param[in] item_flags
2205  *   Bit-fields that holds the items detected until now.
2206  * @param[in] target_protocol
2207  *   The next protocol in the previous item.
2208  * @param[out] error
2209  *   Pointer to error structure.
2210  *
2211  * @return
2212  *   0 on success, a negative errno value otherwise and rte_errno is set.
2213  */
2214 int
2215 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2216                             uint64_t item_flags,
2217                             uint8_t target_protocol,
2218                             const struct rte_flow_item_tcp *flow_mask,
2219                             struct rte_flow_error *error)
2220 {
2221         const struct rte_flow_item_tcp *mask = item->mask;
2222         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2223         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2224                                       MLX5_FLOW_LAYER_OUTER_L3;
2225         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2226                                       MLX5_FLOW_LAYER_OUTER_L4;
2227         int ret;
2228
2229         MLX5_ASSERT(flow_mask);
2230         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2231                 return rte_flow_error_set(error, EINVAL,
2232                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2233                                           "protocol filtering not compatible"
2234                                           " with TCP layer");
2235         if (!(item_flags & l3m))
2236                 return rte_flow_error_set(error, EINVAL,
2237                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2238                                           "L3 is mandatory to filter on L4");
2239         if (item_flags & l4m)
2240                 return rte_flow_error_set(error, EINVAL,
2241                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2242                                           "multiple L4 layers not supported");
2243         if (!mask)
2244                 mask = &rte_flow_item_tcp_mask;
2245         ret = mlx5_flow_item_acceptable
2246                 (item, (const uint8_t *)mask,
2247                  (const uint8_t *)flow_mask,
2248                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2249                  error);
2250         if (ret < 0)
2251                 return ret;
2252         return 0;
2253 }
2254
2255 /**
2256  * Validate VXLAN item.
2257  *
2258  * @param[in] item
2259  *   Item specification.
2260  * @param[in] item_flags
2261  *   Bit-fields that holds the items detected until now.
2262  * @param[in] target_protocol
2263  *   The next protocol in the previous item.
2264  * @param[out] error
2265  *   Pointer to error structure.
2266  *
2267  * @return
2268  *   0 on success, a negative errno value otherwise and rte_errno is set.
2269  */
2270 int
2271 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2272                               uint64_t item_flags,
2273                               struct rte_flow_error *error)
2274 {
2275         const struct rte_flow_item_vxlan *spec = item->spec;
2276         const struct rte_flow_item_vxlan *mask = item->mask;
2277         int ret;
2278         union vni {
2279                 uint32_t vlan_id;
2280                 uint8_t vni[4];
2281         } id = { .vlan_id = 0, };
2282
2283
2284         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2285                 return rte_flow_error_set(error, ENOTSUP,
2286                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2287                                           "multiple tunnel layers not"
2288                                           " supported");
2289         /*
2290          * Verify only UDPv4 is present as defined in
2291          * https://tools.ietf.org/html/rfc7348
2292          */
2293         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2294                 return rte_flow_error_set(error, EINVAL,
2295                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2296                                           "no outer UDP layer found");
2297         if (!mask)
2298                 mask = &rte_flow_item_vxlan_mask;
2299         ret = mlx5_flow_item_acceptable
2300                 (item, (const uint8_t *)mask,
2301                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2302                  sizeof(struct rte_flow_item_vxlan),
2303                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2304         if (ret < 0)
2305                 return ret;
2306         if (spec) {
2307                 memcpy(&id.vni[1], spec->vni, 3);
2308                 memcpy(&id.vni[1], mask->vni, 3);
2309         }
2310         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2311                 return rte_flow_error_set(error, ENOTSUP,
2312                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2313                                           "VXLAN tunnel must be fully defined");
2314         return 0;
2315 }
2316
2317 /**
2318  * Validate VXLAN_GPE item.
2319  *
2320  * @param[in] item
2321  *   Item specification.
2322  * @param[in] item_flags
2323  *   Bit-fields that holds the items detected until now.
2324  * @param[in] priv
2325  *   Pointer to the private data structure.
2326  * @param[in] target_protocol
2327  *   The next protocol in the previous item.
2328  * @param[out] error
2329  *   Pointer to error structure.
2330  *
2331  * @return
2332  *   0 on success, a negative errno value otherwise and rte_errno is set.
2333  */
2334 int
2335 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2336                                   uint64_t item_flags,
2337                                   struct rte_eth_dev *dev,
2338                                   struct rte_flow_error *error)
2339 {
2340         struct mlx5_priv *priv = dev->data->dev_private;
2341         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2342         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2343         int ret;
2344         union vni {
2345                 uint32_t vlan_id;
2346                 uint8_t vni[4];
2347         } id = { .vlan_id = 0, };
2348
2349         if (!priv->config.l3_vxlan_en)
2350                 return rte_flow_error_set(error, ENOTSUP,
2351                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2352                                           "L3 VXLAN is not enabled by device"
2353                                           " parameter and/or not configured in"
2354                                           " firmware");
2355         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2356                 return rte_flow_error_set(error, ENOTSUP,
2357                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2358                                           "multiple tunnel layers not"
2359                                           " supported");
2360         /*
2361          * Verify only UDPv4 is present as defined in
2362          * https://tools.ietf.org/html/rfc7348
2363          */
2364         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2365                 return rte_flow_error_set(error, EINVAL,
2366                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2367                                           "no outer UDP layer found");
2368         if (!mask)
2369                 mask = &rte_flow_item_vxlan_gpe_mask;
2370         ret = mlx5_flow_item_acceptable
2371                 (item, (const uint8_t *)mask,
2372                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2373                  sizeof(struct rte_flow_item_vxlan_gpe),
2374                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2375         if (ret < 0)
2376                 return ret;
2377         if (spec) {
2378                 if (spec->protocol)
2379                         return rte_flow_error_set(error, ENOTSUP,
2380                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2381                                                   item,
2382                                                   "VxLAN-GPE protocol"
2383                                                   " not supported");
2384                 memcpy(&id.vni[1], spec->vni, 3);
2385                 memcpy(&id.vni[1], mask->vni, 3);
2386         }
2387         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2388                 return rte_flow_error_set(error, ENOTSUP,
2389                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2390                                           "VXLAN-GPE tunnel must be fully"
2391                                           " defined");
2392         return 0;
2393 }
2394 /**
2395  * Validate GRE Key item.
2396  *
2397  * @param[in] item
2398  *   Item specification.
2399  * @param[in] item_flags
2400  *   Bit flags to mark detected items.
2401  * @param[in] gre_item
2402  *   Pointer to gre_item
2403  * @param[out] error
2404  *   Pointer to error structure.
2405  *
2406  * @return
2407  *   0 on success, a negative errno value otherwise and rte_errno is set.
2408  */
2409 int
2410 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2411                                 uint64_t item_flags,
2412                                 const struct rte_flow_item *gre_item,
2413                                 struct rte_flow_error *error)
2414 {
2415         const rte_be32_t *mask = item->mask;
2416         int ret = 0;
2417         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2418         const struct rte_flow_item_gre *gre_spec;
2419         const struct rte_flow_item_gre *gre_mask;
2420
2421         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2424                                           "Multiple GRE key not support");
2425         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2426                 return rte_flow_error_set(error, ENOTSUP,
2427                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2428                                           "No preceding GRE header");
2429         if (item_flags & MLX5_FLOW_LAYER_INNER)
2430                 return rte_flow_error_set(error, ENOTSUP,
2431                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2432                                           "GRE key following a wrong item");
2433         gre_mask = gre_item->mask;
2434         if (!gre_mask)
2435                 gre_mask = &rte_flow_item_gre_mask;
2436         gre_spec = gre_item->spec;
2437         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2438                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2439                 return rte_flow_error_set(error, EINVAL,
2440                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2441                                           "Key bit must be on");
2442
2443         if (!mask)
2444                 mask = &gre_key_default_mask;
2445         ret = mlx5_flow_item_acceptable
2446                 (item, (const uint8_t *)mask,
2447                  (const uint8_t *)&gre_key_default_mask,
2448                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2449         return ret;
2450 }
2451
2452 /**
2453  * Validate GRE item.
2454  *
2455  * @param[in] item
2456  *   Item specification.
2457  * @param[in] item_flags
2458  *   Bit flags to mark detected items.
2459  * @param[in] target_protocol
2460  *   The next protocol in the previous item.
2461  * @param[out] error
2462  *   Pointer to error structure.
2463  *
2464  * @return
2465  *   0 on success, a negative errno value otherwise and rte_errno is set.
2466  */
2467 int
2468 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2469                             uint64_t item_flags,
2470                             uint8_t target_protocol,
2471                             struct rte_flow_error *error)
2472 {
2473         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2474         const struct rte_flow_item_gre *mask = item->mask;
2475         int ret;
2476         const struct rte_flow_item_gre nic_mask = {
2477                 .c_rsvd0_ver = RTE_BE16(0xB000),
2478                 .protocol = RTE_BE16(UINT16_MAX),
2479         };
2480
2481         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2484                                           "protocol filtering not compatible"
2485                                           " with this GRE layer");
2486         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2487                 return rte_flow_error_set(error, ENOTSUP,
2488                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2489                                           "multiple tunnel layers not"
2490                                           " supported");
2491         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2492                 return rte_flow_error_set(error, ENOTSUP,
2493                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2494                                           "L3 Layer is missing");
2495         if (!mask)
2496                 mask = &rte_flow_item_gre_mask;
2497         ret = mlx5_flow_item_acceptable
2498                 (item, (const uint8_t *)mask,
2499                  (const uint8_t *)&nic_mask,
2500                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2501                  error);
2502         if (ret < 0)
2503                 return ret;
2504 #ifndef HAVE_MLX5DV_DR
2505 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2506         if (spec && (spec->protocol & mask->protocol))
2507                 return rte_flow_error_set(error, ENOTSUP,
2508                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2509                                           "without MPLS support the"
2510                                           " specification cannot be used for"
2511                                           " filtering");
2512 #endif
2513 #endif
2514         return 0;
2515 }
2516
2517 /**
2518  * Validate Geneve item.
2519  *
2520  * @param[in] item
2521  *   Item specification.
2522  * @param[in] itemFlags
2523  *   Bit-fields that holds the items detected until now.
2524  * @param[in] enPriv
2525  *   Pointer to the private data structure.
2526  * @param[out] error
2527  *   Pointer to error structure.
2528  *
2529  * @return
2530  *   0 on success, a negative errno value otherwise and rte_errno is set.
2531  */
2532
2533 int
2534 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2535                                uint64_t item_flags,
2536                                struct rte_eth_dev *dev,
2537                                struct rte_flow_error *error)
2538 {
2539         struct mlx5_priv *priv = dev->data->dev_private;
2540         const struct rte_flow_item_geneve *spec = item->spec;
2541         const struct rte_flow_item_geneve *mask = item->mask;
2542         int ret;
2543         uint16_t gbhdr;
2544         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2545                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2546         const struct rte_flow_item_geneve nic_mask = {
2547                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2548                 .vni = "\xff\xff\xff",
2549                 .protocol = RTE_BE16(UINT16_MAX),
2550         };
2551
2552         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2553                 return rte_flow_error_set(error, ENOTSUP,
2554                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2555                                           "L3 Geneve is not enabled by device"
2556                                           " parameter and/or not configured in"
2557                                           " firmware");
2558         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2559                 return rte_flow_error_set(error, ENOTSUP,
2560                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2561                                           "multiple tunnel layers not"
2562                                           " supported");
2563         /*
2564          * Verify only UDPv4 is present as defined in
2565          * https://tools.ietf.org/html/rfc7348
2566          */
2567         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2568                 return rte_flow_error_set(error, EINVAL,
2569                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2570                                           "no outer UDP layer found");
2571         if (!mask)
2572                 mask = &rte_flow_item_geneve_mask;
2573         ret = mlx5_flow_item_acceptable
2574                                   (item, (const uint8_t *)mask,
2575                                    (const uint8_t *)&nic_mask,
2576                                    sizeof(struct rte_flow_item_geneve),
2577                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2578         if (ret)
2579                 return ret;
2580         if (spec) {
2581                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2582                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2583                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2584                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2585                         return rte_flow_error_set(error, ENOTSUP,
2586                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2587                                                   item,
2588                                                   "Geneve protocol unsupported"
2589                                                   " fields are being used");
2590                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2591                         return rte_flow_error_set
2592                                         (error, ENOTSUP,
2593                                          RTE_FLOW_ERROR_TYPE_ITEM,
2594                                          item,
2595                                          "Unsupported Geneve options length");
2596         }
2597         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2598                 return rte_flow_error_set
2599                                     (error, ENOTSUP,
2600                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2601                                      "Geneve tunnel must be fully defined");
2602         return 0;
2603 }
2604
2605 /**
2606  * Validate MPLS item.
2607  *
2608  * @param[in] dev
2609  *   Pointer to the rte_eth_dev structure.
2610  * @param[in] item
2611  *   Item specification.
2612  * @param[in] item_flags
2613  *   Bit-fields that holds the items detected until now.
2614  * @param[in] prev_layer
2615  *   The protocol layer indicated in previous item.
2616  * @param[out] error
2617  *   Pointer to error structure.
2618  *
2619  * @return
2620  *   0 on success, a negative errno value otherwise and rte_errno is set.
2621  */
2622 int
2623 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2624                              const struct rte_flow_item *item __rte_unused,
2625                              uint64_t item_flags __rte_unused,
2626                              uint64_t prev_layer __rte_unused,
2627                              struct rte_flow_error *error)
2628 {
2629 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2630         const struct rte_flow_item_mpls *mask = item->mask;
2631         struct mlx5_priv *priv = dev->data->dev_private;
2632         int ret;
2633
2634         if (!priv->config.mpls_en)
2635                 return rte_flow_error_set(error, ENOTSUP,
2636                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2637                                           "MPLS not supported or"
2638                                           " disabled in firmware"
2639                                           " configuration.");
2640         /* MPLS over IP, UDP, GRE is allowed */
2641         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2642                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2643                             MLX5_FLOW_LAYER_GRE |
2644                             MLX5_FLOW_LAYER_GRE_KEY)))
2645                 return rte_flow_error_set(error, EINVAL,
2646                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2647                                           "protocol filtering not compatible"
2648                                           " with MPLS layer");
2649         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2650         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2651             !(item_flags & MLX5_FLOW_LAYER_GRE))
2652                 return rte_flow_error_set(error, ENOTSUP,
2653                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2654                                           "multiple tunnel layers not"
2655                                           " supported");
2656         if (!mask)
2657                 mask = &rte_flow_item_mpls_mask;
2658         ret = mlx5_flow_item_acceptable
2659                 (item, (const uint8_t *)mask,
2660                  (const uint8_t *)&rte_flow_item_mpls_mask,
2661                  sizeof(struct rte_flow_item_mpls),
2662                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2663         if (ret < 0)
2664                 return ret;
2665         return 0;
2666 #else
2667         return rte_flow_error_set(error, ENOTSUP,
2668                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2669                                   "MPLS is not supported by Verbs, please"
2670                                   " update.");
2671 #endif
2672 }
2673
2674 /**
2675  * Validate NVGRE item.
2676  *
2677  * @param[in] item
2678  *   Item specification.
2679  * @param[in] item_flags
2680  *   Bit flags to mark detected items.
2681  * @param[in] target_protocol
2682  *   The next protocol in the previous item.
2683  * @param[out] error
2684  *   Pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success, a negative errno value otherwise and rte_errno is set.
2688  */
2689 int
2690 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2691                               uint64_t item_flags,
2692                               uint8_t target_protocol,
2693                               struct rte_flow_error *error)
2694 {
2695         const struct rte_flow_item_nvgre *mask = item->mask;
2696         int ret;
2697
2698         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2699                 return rte_flow_error_set(error, EINVAL,
2700                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2701                                           "protocol filtering not compatible"
2702                                           " with this GRE layer");
2703         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2704                 return rte_flow_error_set(error, ENOTSUP,
2705                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2706                                           "multiple tunnel layers not"
2707                                           " supported");
2708         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2709                 return rte_flow_error_set(error, ENOTSUP,
2710                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2711                                           "L3 Layer is missing");
2712         if (!mask)
2713                 mask = &rte_flow_item_nvgre_mask;
2714         ret = mlx5_flow_item_acceptable
2715                 (item, (const uint8_t *)mask,
2716                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2717                  sizeof(struct rte_flow_item_nvgre),
2718                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2719         if (ret < 0)
2720                 return ret;
2721         return 0;
2722 }
2723
2724 /**
2725  * Validate eCPRI item.
2726  *
2727  * @param[in] item
2728  *   Item specification.
2729  * @param[in] item_flags
2730  *   Bit-fields that holds the items detected until now.
2731  * @param[in] last_item
2732  *   Previous validated item in the pattern items.
2733  * @param[in] ether_type
2734  *   Type in the ethernet layer header (including dot1q).
2735  * @param[in] acc_mask
2736  *   Acceptable mask, if NULL default internal default mask
2737  *   will be used to check whether item fields are supported.
2738  * @param[out] error
2739  *   Pointer to error structure.
2740  *
2741  * @return
2742  *   0 on success, a negative errno value otherwise and rte_errno is set.
2743  */
2744 int
2745 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2746                               uint64_t item_flags,
2747                               uint64_t last_item,
2748                               uint16_t ether_type,
2749                               const struct rte_flow_item_ecpri *acc_mask,
2750                               struct rte_flow_error *error)
2751 {
2752         const struct rte_flow_item_ecpri *mask = item->mask;
2753         const struct rte_flow_item_ecpri nic_mask = {
2754                 .hdr = {
2755                         .common = {
2756                                 .u32 =
2757                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2758                                         .type = 0xFF,
2759                                         }).u32),
2760                         },
2761                         .dummy[0] = 0xFFFFFFFF,
2762                 },
2763         };
2764         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2765                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2766         struct rte_flow_item_ecpri mask_lo;
2767
2768         if (!(last_item & outer_l2_vlan) &&
2769             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
2770                 return rte_flow_error_set(error, EINVAL,
2771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
2773         if ((last_item & outer_l2_vlan) && ether_type &&
2774             ether_type != RTE_ETHER_TYPE_ECPRI)
2775                 return rte_flow_error_set(error, EINVAL,
2776                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2777                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
2778         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2779                 return rte_flow_error_set(error, EINVAL,
2780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2781                                           "eCPRI with tunnel is not supported right now");
2782         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2783                 return rte_flow_error_set(error, ENOTSUP,
2784                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2785                                           "multiple L3 layers not supported");
2786         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2787                 return rte_flow_error_set(error, EINVAL,
2788                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2789                                           "eCPRI cannot coexist with a TCP layer");
2790         /* In specification, eCPRI could be over UDP layer. */
2791         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2792                 return rte_flow_error_set(error, EINVAL,
2793                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2794                                           "eCPRI over UDP layer is not yet supported right now");
2795         /* Mask for type field in common header could be zero. */
2796         if (!mask)
2797                 mask = &rte_flow_item_ecpri_mask;
2798         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2799         /* Input mask is in big-endian format. */
2800         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2801                 return rte_flow_error_set(error, EINVAL,
2802                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2803                                           "partial mask is not supported for protocol");
2804         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2805                 return rte_flow_error_set(error, EINVAL,
2806                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2807                                           "message header mask must be after a type mask");
2808         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2809                                          acc_mask ? (const uint8_t *)acc_mask
2810                                                   : (const uint8_t *)&nic_mask,
2811                                          sizeof(struct rte_flow_item_ecpri),
2812                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2813 }
2814
2815 /**
2816  * Release resource related QUEUE/RSS action split.
2817  *
2818  * @param dev
2819  *   Pointer to Ethernet device.
2820  * @param flow
2821  *   Flow to release id's from.
2822  */
2823 static void
2824 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2825                              struct rte_flow *flow)
2826 {
2827         struct mlx5_priv *priv = dev->data->dev_private;
2828         uint32_t handle_idx;
2829         struct mlx5_flow_handle *dev_handle;
2830
2831         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2832                        handle_idx, dev_handle, next)
2833                 if (dev_handle->split_flow_id)
2834                         mlx5_ipool_free(priv->sh->ipool
2835                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
2836                                         dev_handle->split_flow_id);
2837 }
2838
2839 static int
2840 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2841                    const struct rte_flow_attr *attr __rte_unused,
2842                    const struct rte_flow_item items[] __rte_unused,
2843                    const struct rte_flow_action actions[] __rte_unused,
2844                    bool external __rte_unused,
2845                    int hairpin __rte_unused,
2846                    struct rte_flow_error *error)
2847 {
2848         return rte_flow_error_set(error, ENOTSUP,
2849                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2850 }
2851
2852 static struct mlx5_flow *
2853 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2854                   const struct rte_flow_attr *attr __rte_unused,
2855                   const struct rte_flow_item items[] __rte_unused,
2856                   const struct rte_flow_action actions[] __rte_unused,
2857                   struct rte_flow_error *error)
2858 {
2859         rte_flow_error_set(error, ENOTSUP,
2860                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2861         return NULL;
2862 }
2863
2864 static int
2865 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2866                     struct mlx5_flow *dev_flow __rte_unused,
2867                     const struct rte_flow_attr *attr __rte_unused,
2868                     const struct rte_flow_item items[] __rte_unused,
2869                     const struct rte_flow_action actions[] __rte_unused,
2870                     struct rte_flow_error *error)
2871 {
2872         return rte_flow_error_set(error, ENOTSUP,
2873                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2874 }
2875
2876 static int
2877 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2878                 struct rte_flow *flow __rte_unused,
2879                 struct rte_flow_error *error)
2880 {
2881         return rte_flow_error_set(error, ENOTSUP,
2882                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2883 }
2884
2885 static void
2886 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2887                  struct rte_flow *flow __rte_unused)
2888 {
2889 }
2890
2891 static void
2892 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2893                   struct rte_flow *flow __rte_unused)
2894 {
2895 }
2896
2897 static int
2898 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2899                 struct rte_flow *flow __rte_unused,
2900                 const struct rte_flow_action *actions __rte_unused,
2901                 void *data __rte_unused,
2902                 struct rte_flow_error *error)
2903 {
2904         return rte_flow_error_set(error, ENOTSUP,
2905                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2906 }
2907
2908 static int
2909 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
2910                       uint32_t domains __rte_unused,
2911                       uint32_t flags __rte_unused)
2912 {
2913         return 0;
2914 }
2915
2916 /* Void driver to protect from null pointer reference. */
2917 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2918         .validate = flow_null_validate,
2919         .prepare = flow_null_prepare,
2920         .translate = flow_null_translate,
2921         .apply = flow_null_apply,
2922         .remove = flow_null_remove,
2923         .destroy = flow_null_destroy,
2924         .query = flow_null_query,
2925         .sync_domain = flow_null_sync_domain,
2926 };
2927
2928 /**
2929  * Select flow driver type according to flow attributes and device
2930  * configuration.
2931  *
2932  * @param[in] dev
2933  *   Pointer to the dev structure.
2934  * @param[in] attr
2935  *   Pointer to the flow attributes.
2936  *
2937  * @return
2938  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2939  */
2940 static enum mlx5_flow_drv_type
2941 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2942 {
2943         struct mlx5_priv *priv = dev->data->dev_private;
2944         /* The OS can determine first a specific flow type (DV, VERBS) */
2945         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2946
2947         if (type != MLX5_FLOW_TYPE_MAX)
2948                 return type;
2949         /* If no OS specific type - continue with DV/VERBS selection */
2950         if (attr->transfer && priv->config.dv_esw_en)
2951                 type = MLX5_FLOW_TYPE_DV;
2952         if (!attr->transfer)
2953                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2954                                                  MLX5_FLOW_TYPE_VERBS;
2955         return type;
2956 }
2957
2958 #define flow_get_drv_ops(type) flow_drv_ops[type]
2959
2960 /**
2961  * Flow driver validation API. This abstracts calling driver specific functions.
2962  * The type of flow driver is determined according to flow attributes.
2963  *
2964  * @param[in] dev
2965  *   Pointer to the dev structure.
2966  * @param[in] attr
2967  *   Pointer to the flow attributes.
2968  * @param[in] items
2969  *   Pointer to the list of items.
2970  * @param[in] actions
2971  *   Pointer to the list of actions.
2972  * @param[in] external
2973  *   This flow rule is created by request external to PMD.
2974  * @param[in] hairpin
2975  *   Number of hairpin TX actions, 0 means classic flow.
2976  * @param[out] error
2977  *   Pointer to the error structure.
2978  *
2979  * @return
2980  *   0 on success, a negative errno value otherwise and rte_errno is set.
2981  */
2982 static inline int
2983 flow_drv_validate(struct rte_eth_dev *dev,
2984                   const struct rte_flow_attr *attr,
2985                   const struct rte_flow_item items[],
2986                   const struct rte_flow_action actions[],
2987                   bool external, int hairpin, struct rte_flow_error *error)
2988 {
2989         const struct mlx5_flow_driver_ops *fops;
2990         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2991
2992         fops = flow_get_drv_ops(type);
2993         return fops->validate(dev, attr, items, actions, external,
2994                               hairpin, error);
2995 }
2996
2997 /**
2998  * Flow driver preparation API. This abstracts calling driver specific
2999  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3000  * calculates the size of memory required for device flow, allocates the memory,
3001  * initializes the device flow and returns the pointer.
3002  *
3003  * @note
3004  *   This function initializes device flow structure such as dv or verbs in
3005  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3006  *   rest. For example, adding returning device flow to flow->dev_flow list and
3007  *   setting backward reference to the flow should be done out of this function.
3008  *   layers field is not filled either.
3009  *
3010  * @param[in] dev
3011  *   Pointer to the dev structure.
3012  * @param[in] attr
3013  *   Pointer to the flow attributes.
3014  * @param[in] items
3015  *   Pointer to the list of items.
3016  * @param[in] actions
3017  *   Pointer to the list of actions.
3018  * @param[in] flow_idx
3019  *   This memory pool index to the flow.
3020  * @param[out] error
3021  *   Pointer to the error structure.
3022  *
3023  * @return
3024  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3025  */
3026 static inline struct mlx5_flow *
3027 flow_drv_prepare(struct rte_eth_dev *dev,
3028                  const struct rte_flow *flow,
3029                  const struct rte_flow_attr *attr,
3030                  const struct rte_flow_item items[],
3031                  const struct rte_flow_action actions[],
3032                  uint32_t flow_idx,
3033                  struct rte_flow_error *error)
3034 {
3035         const struct mlx5_flow_driver_ops *fops;
3036         enum mlx5_flow_drv_type type = flow->drv_type;
3037         struct mlx5_flow *mlx5_flow = NULL;
3038
3039         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3040         fops = flow_get_drv_ops(type);
3041         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3042         if (mlx5_flow)
3043                 mlx5_flow->flow_idx = flow_idx;
3044         return mlx5_flow;
3045 }
3046
3047 /**
3048  * Flow driver translation API. This abstracts calling driver specific
3049  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3050  * translates a generic flow into a driver flow. flow_drv_prepare() must
3051  * precede.
3052  *
3053  * @note
3054  *   dev_flow->layers could be filled as a result of parsing during translation
3055  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3056  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3057  *   flow->actions could be overwritten even though all the expanded dev_flows
3058  *   have the same actions.
3059  *
3060  * @param[in] dev
3061  *   Pointer to the rte dev structure.
3062  * @param[in, out] dev_flow
3063  *   Pointer to the mlx5 flow.
3064  * @param[in] attr
3065  *   Pointer to the flow attributes.
3066  * @param[in] items
3067  *   Pointer to the list of items.
3068  * @param[in] actions
3069  *   Pointer to the list of actions.
3070  * @param[out] error
3071  *   Pointer to the error structure.
3072  *
3073  * @return
3074  *   0 on success, a negative errno value otherwise and rte_errno is set.
3075  */
3076 static inline int
3077 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3078                    const struct rte_flow_attr *attr,
3079                    const struct rte_flow_item items[],
3080                    const struct rte_flow_action actions[],
3081                    struct rte_flow_error *error)
3082 {
3083         const struct mlx5_flow_driver_ops *fops;
3084         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3085
3086         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3087         fops = flow_get_drv_ops(type);
3088         return fops->translate(dev, dev_flow, attr, items, actions, error);
3089 }
3090
3091 /**
3092  * Flow driver apply API. This abstracts calling driver specific functions.
3093  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3094  * translated driver flows on to device. flow_drv_translate() must precede.
3095  *
3096  * @param[in] dev
3097  *   Pointer to Ethernet device structure.
3098  * @param[in, out] flow
3099  *   Pointer to flow structure.
3100  * @param[out] error
3101  *   Pointer to error structure.
3102  *
3103  * @return
3104  *   0 on success, a negative errno value otherwise and rte_errno is set.
3105  */
3106 static inline int
3107 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3108                struct rte_flow_error *error)
3109 {
3110         const struct mlx5_flow_driver_ops *fops;
3111         enum mlx5_flow_drv_type type = flow->drv_type;
3112
3113         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3114         fops = flow_get_drv_ops(type);
3115         return fops->apply(dev, flow, error);
3116 }
3117
3118 /**
3119  * Flow driver destroy API. This abstracts calling driver specific functions.
3120  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3121  * on device and releases resources of the flow.
3122  *
3123  * @param[in] dev
3124  *   Pointer to Ethernet device.
3125  * @param[in, out] flow
3126  *   Pointer to flow structure.
3127  */
3128 static inline void
3129 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3130 {
3131         const struct mlx5_flow_driver_ops *fops;
3132         enum mlx5_flow_drv_type type = flow->drv_type;
3133
3134         flow_mreg_split_qrss_release(dev, flow);
3135         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3136         fops = flow_get_drv_ops(type);
3137         fops->destroy(dev, flow);
3138 }
3139
3140 /**
3141  * Get RSS action from the action list.
3142  *
3143  * @param[in] actions
3144  *   Pointer to the list of actions.
3145  *
3146  * @return
3147  *   Pointer to the RSS action if exist, else return NULL.
3148  */
3149 static const struct rte_flow_action_rss*
3150 flow_get_rss_action(const struct rte_flow_action actions[])
3151 {
3152         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3153                 switch (actions->type) {
3154                 case RTE_FLOW_ACTION_TYPE_RSS:
3155                         return (const struct rte_flow_action_rss *)
3156                                actions->conf;
3157                 default:
3158                         break;
3159                 }
3160         }
3161         return NULL;
3162 }
3163
3164 /**
3165  * Get ASO age action by index.
3166  *
3167  * @param[in] dev
3168  *   Pointer to the Ethernet device structure.
3169  * @param[in] age_idx
3170  *   Index to the ASO age action.
3171  *
3172  * @return
3173  *   The specified ASO age action.
3174  */
3175 struct mlx5_aso_age_action*
3176 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3177 {
3178         uint16_t pool_idx = age_idx & UINT16_MAX;
3179         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3180         struct mlx5_priv *priv = dev->data->dev_private;
3181         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3182         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3183
3184         return &pool->actions[offset - 1];
3185 }
3186
3187 /* maps shared action to translated non shared in some actions array */
3188 struct mlx5_translated_shared_action {
3189         struct rte_flow_shared_action *action; /**< Shared action */
3190         int index; /**< Index in related array of rte_flow_action */
3191 };
3192
3193 /**
3194  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3195  * non shared action if translation possible.
3196  * This functionality used to run same execution path for both shared & non
3197  * shared actions on flow create. All necessary preparations for shared
3198  * action handling should be preformed on *shared* actions list returned
3199  * from this call.
3200  *
3201  * @param[in] dev
3202  *   Pointer to Ethernet device.
3203  * @param[in] actions
3204  *   List of actions to translate.
3205  * @param[out] shared
3206  *   List to store translated shared actions.
3207  * @param[in, out] shared_n
3208  *   Size of *shared* array. On return should be updated with number of shared
3209  *   actions retrieved from the *actions* list.
3210  * @param[out] translated_actions
3211  *   List of actions where all shared actions were translated to non shared
3212  *   if possible. NULL if no translation took place.
3213  * @param[out] error
3214  *   Pointer to the error structure.
3215  *
3216  * @return
3217  *   0 on success, a negative errno value otherwise and rte_errno is set.
3218  */
3219 static int
3220 flow_shared_actions_translate(struct rte_eth_dev *dev,
3221                               const struct rte_flow_action actions[],
3222                               struct mlx5_translated_shared_action *shared,
3223                               int *shared_n,
3224                               struct rte_flow_action **translated_actions,
3225                               struct rte_flow_error *error)
3226 {
3227         struct mlx5_priv *priv = dev->data->dev_private;
3228         struct rte_flow_action *translated = NULL;
3229         size_t actions_size;
3230         int n;
3231         int copied_n = 0;
3232         struct mlx5_translated_shared_action *shared_end = NULL;
3233
3234         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3235                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3236                         continue;
3237                 if (copied_n == *shared_n) {
3238                         return rte_flow_error_set
3239                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3240                                  NULL, "too many shared actions");
3241                 }
3242                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3243                            sizeof(actions[n].conf));
3244                 shared[copied_n].index = n;
3245                 copied_n++;
3246         }
3247         n++;
3248         *shared_n = copied_n;
3249         if (!copied_n)
3250                 return 0;
3251         actions_size = sizeof(struct rte_flow_action) * n;
3252         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3253         if (!translated) {
3254                 rte_errno = ENOMEM;
3255                 return -ENOMEM;
3256         }
3257         memcpy(translated, actions, actions_size);
3258         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3259                 struct mlx5_shared_action_rss *shared_rss;
3260                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3261                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3262                 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET)
3263                                                                            - 1);
3264
3265                 switch (type) {
3266                 case MLX5_SHARED_ACTION_TYPE_RSS:
3267                         shared_rss = mlx5_ipool_get
3268                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3269                         translated[shared->index].type =
3270                                 RTE_FLOW_ACTION_TYPE_RSS;
3271                         translated[shared->index].conf =
3272                                 &shared_rss->origin;
3273                         break;
3274                 case MLX5_SHARED_ACTION_TYPE_AGE:
3275                         if (priv->sh->flow_hit_aso_en) {
3276                                 translated[shared->index].type =
3277                                         (enum rte_flow_action_type)
3278                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3279                                 translated[shared->index].conf =
3280                                                          (void *)(uintptr_t)idx;
3281                                 break;
3282                         }
3283                         /* Fall-through */
3284                 default:
3285                         mlx5_free(translated);
3286                         return rte_flow_error_set
3287                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3288                                  NULL, "invalid shared action type");
3289                 }
3290         }
3291         *translated_actions = translated;
3292         return 0;
3293 }
3294
3295 /**
3296  * Get Shared RSS action from the action list.
3297  *
3298  * @param[in] dev
3299  *   Pointer to Ethernet device.
3300  * @param[in] shared
3301  *   Pointer to the list of actions.
3302  * @param[in] shared_n
3303  *   Actions list length.
3304  *
3305  * @return
3306  *   The MLX5 RSS action ID if exists, otherwise return 0.
3307  */
3308 static uint32_t
3309 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3310                            struct mlx5_translated_shared_action *shared,
3311                            int shared_n)
3312 {
3313         struct mlx5_translated_shared_action *shared_end;
3314         struct mlx5_priv *priv = dev->data->dev_private;
3315         struct mlx5_shared_action_rss *shared_rss;
3316
3317
3318         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3319                 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action;
3320                 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET;
3321                 uint32_t idx = act_idx &
3322                                    ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1);
3323                 switch (type) {
3324                 case MLX5_SHARED_ACTION_TYPE_RSS:
3325                         shared_rss = mlx5_ipool_get
3326                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3327                                                                            idx);
3328                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3329                                            __ATOMIC_RELAXED);
3330                         return idx;
3331                 default:
3332                         break;
3333                 }
3334         }
3335         return 0;
3336 }
3337
3338 static unsigned int
3339 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3340 {
3341         const struct rte_flow_item *item;
3342         unsigned int has_vlan = 0;
3343
3344         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3345                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3346                         has_vlan = 1;
3347                         break;
3348                 }
3349         }
3350         if (has_vlan)
3351                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3352                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3353         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3354                                MLX5_EXPANSION_ROOT_OUTER;
3355 }
3356
3357 /**
3358  *  Get layer flags from the prefix flow.
3359  *
3360  *  Some flows may be split to several subflows, the prefix subflow gets the
3361  *  match items and the suffix sub flow gets the actions.
3362  *  Some actions need the user defined match item flags to get the detail for
3363  *  the action.
3364  *  This function helps the suffix flow to get the item layer flags from prefix
3365  *  subflow.
3366  *
3367  * @param[in] dev_flow
3368  *   Pointer the created preifx subflow.
3369  *
3370  * @return
3371  *   The layers get from prefix subflow.
3372  */
3373 static inline uint64_t
3374 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3375 {
3376         uint64_t layers = 0;
3377
3378         /*
3379          * Layers bits could be localization, but usually the compiler will
3380          * help to do the optimization work for source code.
3381          * If no decap actions, use the layers directly.
3382          */
3383         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3384                 return dev_flow->handle->layers;
3385         /* Convert L3 layers with decap action. */
3386         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3387                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3388         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3389                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3390         /* Convert L4 layers with decap action.  */
3391         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3392                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3393         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3394                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3395         return layers;
3396 }
3397
3398 /**
3399  * Get metadata split action information.
3400  *
3401  * @param[in] actions
3402  *   Pointer to the list of actions.
3403  * @param[out] qrss
3404  *   Pointer to the return pointer.
3405  * @param[out] qrss_type
3406  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3407  *   if no QUEUE/RSS is found.
3408  * @param[out] encap_idx
3409  *   Pointer to the index of the encap action if exists, otherwise the last
3410  *   action index.
3411  *
3412  * @return
3413  *   Total number of actions.
3414  */
3415 static int
3416 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3417                                        const struct rte_flow_action **qrss,
3418                                        int *encap_idx)
3419 {
3420         const struct rte_flow_action_raw_encap *raw_encap;
3421         int actions_n = 0;
3422         int raw_decap_idx = -1;
3423
3424         *encap_idx = -1;
3425         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3426                 switch (actions->type) {
3427                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3428                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3429                         *encap_idx = actions_n;
3430                         break;
3431                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3432                         raw_decap_idx = actions_n;
3433                         break;
3434                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3435                         raw_encap = actions->conf;
3436                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3437                                 *encap_idx = raw_decap_idx != -1 ?
3438                                                       raw_decap_idx : actions_n;
3439                         break;
3440                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3441                 case RTE_FLOW_ACTION_TYPE_RSS:
3442                         *qrss = actions;
3443                         break;
3444                 default:
3445                         break;
3446                 }
3447                 actions_n++;
3448         }
3449         if (*encap_idx == -1)
3450                 *encap_idx = actions_n;
3451         /* Count RTE_FLOW_ACTION_TYPE_END. */
3452         return actions_n + 1;
3453 }
3454
3455 /**
3456  * Check meter action from the action list.
3457  *
3458  * @param[in] actions
3459  *   Pointer to the list of actions.
3460  * @param[out] mtr
3461  *   Pointer to the meter exist flag.
3462  *
3463  * @return
3464  *   Total number of actions.
3465  */
3466 static int
3467 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3468 {
3469         int actions_n = 0;
3470
3471         MLX5_ASSERT(mtr);
3472         *mtr = 0;
3473         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3474                 switch (actions->type) {
3475                 case RTE_FLOW_ACTION_TYPE_METER:
3476                         *mtr = 1;
3477                         break;
3478                 default:
3479                         break;
3480                 }
3481                 actions_n++;
3482         }
3483         /* Count RTE_FLOW_ACTION_TYPE_END. */
3484         return actions_n + 1;
3485 }
3486
3487 /**
3488  * Check if the flow should be split due to hairpin.
3489  * The reason for the split is that in current HW we can't
3490  * support encap and push-vlan on Rx, so if a flow contains
3491  * these actions we move it to Tx.
3492  *
3493  * @param dev
3494  *   Pointer to Ethernet device.
3495  * @param[in] attr
3496  *   Flow rule attributes.
3497  * @param[in] actions
3498  *   Associated actions (list terminated by the END action).
3499  *
3500  * @return
3501  *   > 0 the number of actions and the flow should be split,
3502  *   0 when no split required.
3503  */
3504 static int
3505 flow_check_hairpin_split(struct rte_eth_dev *dev,
3506                          const struct rte_flow_attr *attr,
3507                          const struct rte_flow_action actions[])
3508 {
3509         int queue_action = 0;
3510         int action_n = 0;
3511         int split = 0;
3512         const struct rte_flow_action_queue *queue;
3513         const struct rte_flow_action_rss *rss;
3514         const struct rte_flow_action_raw_encap *raw_encap;
3515         const struct rte_eth_hairpin_conf *conf;
3516
3517         if (!attr->ingress)
3518                 return 0;
3519         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3520                 switch (actions->type) {
3521                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3522                         queue = actions->conf;
3523                         if (queue == NULL)
3524                                 return 0;
3525                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3526                         if (conf != NULL && !!conf->tx_explicit)
3527                                 return 0;
3528                         queue_action = 1;
3529                         action_n++;
3530                         break;
3531                 case RTE_FLOW_ACTION_TYPE_RSS:
3532                         rss = actions->conf;
3533                         if (rss == NULL || rss->queue_num == 0)
3534                                 return 0;
3535                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3536                         if (conf != NULL && !!conf->tx_explicit)
3537                                 return 0;
3538                         queue_action = 1;
3539                         action_n++;
3540                         break;
3541                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3542                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3543                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3544                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3545                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3546                         split++;
3547                         action_n++;
3548                         break;
3549                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3550                         raw_encap = actions->conf;
3551                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3552                                 split++;
3553                         action_n++;
3554                         break;
3555                 default:
3556                         action_n++;
3557                         break;
3558                 }
3559         }
3560         if (split && queue_action)
3561                 return action_n;
3562         return 0;
3563 }
3564
3565 /* Declare flow create/destroy prototype in advance. */
3566 static uint32_t
3567 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3568                  const struct rte_flow_attr *attr,
3569                  const struct rte_flow_item items[],
3570                  const struct rte_flow_action actions[],
3571                  bool external, struct rte_flow_error *error);
3572
3573 static void
3574 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3575                   uint32_t flow_idx);
3576
3577 struct mlx5_hlist_entry *
3578 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
3579                        void *cb_ctx)
3580 {
3581         struct rte_eth_dev *dev = list->ctx;
3582         struct mlx5_priv *priv = dev->data->dev_private;
3583         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3584         struct mlx5_flow_mreg_copy_resource *mcp_res;
3585         struct rte_flow_error *error = ctx->error;
3586         uint32_t idx = 0;
3587         int ret;
3588         uint32_t mark_id = key;
3589         struct rte_flow_attr attr = {
3590                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3591                 .ingress = 1,
3592         };
3593         struct mlx5_rte_flow_item_tag tag_spec = {
3594                 .data = mark_id,
3595         };
3596         struct rte_flow_item items[] = {
3597                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3598         };
3599         struct rte_flow_action_mark ftag = {
3600                 .id = mark_id,
3601         };
3602         struct mlx5_flow_action_copy_mreg cp_mreg = {
3603                 .dst = REG_B,
3604                 .src = REG_NON,
3605         };
3606         struct rte_flow_action_jump jump = {
3607                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3608         };
3609         struct rte_flow_action actions[] = {
3610                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3611         };
3612
3613         /* Fill the register fileds in the flow. */
3614         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3615         if (ret < 0)
3616                 return NULL;
3617         tag_spec.id = ret;
3618         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3619         if (ret < 0)
3620                 return NULL;
3621         cp_mreg.src = ret;
3622         /* Provide the full width of FLAG specific value. */
3623         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3624                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3625         /* Build a new flow. */
3626         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3627                 items[0] = (struct rte_flow_item){
3628                         .type = (enum rte_flow_item_type)
3629                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3630                         .spec = &tag_spec,
3631                 };
3632                 items[1] = (struct rte_flow_item){
3633                         .type = RTE_FLOW_ITEM_TYPE_END,
3634                 };
3635                 actions[0] = (struct rte_flow_action){
3636                         .type = (enum rte_flow_action_type)
3637                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3638                         .conf = &ftag,
3639                 };
3640                 actions[1] = (struct rte_flow_action){
3641                         .type = (enum rte_flow_action_type)
3642                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3643                         .conf = &cp_mreg,
3644                 };
3645                 actions[2] = (struct rte_flow_action){
3646                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3647                         .conf = &jump,
3648                 };
3649                 actions[3] = (struct rte_flow_action){
3650                         .type = RTE_FLOW_ACTION_TYPE_END,
3651                 };
3652         } else {
3653                 /* Default rule, wildcard match. */
3654                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3655                 items[0] = (struct rte_flow_item){
3656                         .type = RTE_FLOW_ITEM_TYPE_END,
3657                 };
3658                 actions[0] = (struct rte_flow_action){
3659                         .type = (enum rte_flow_action_type)
3660                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3661                         .conf = &cp_mreg,
3662                 };
3663                 actions[1] = (struct rte_flow_action){
3664                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3665                         .conf = &jump,
3666                 };
3667                 actions[2] = (struct rte_flow_action){
3668                         .type = RTE_FLOW_ACTION_TYPE_END,
3669                 };
3670         }
3671         /* Build a new entry. */
3672         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3673         if (!mcp_res) {
3674                 rte_errno = ENOMEM;
3675                 return NULL;
3676         }
3677         mcp_res->idx = idx;
3678         /*
3679          * The copy Flows are not included in any list. There
3680          * ones are referenced from other Flows and can not
3681          * be applied, removed, deleted in ardbitrary order
3682          * by list traversing.
3683          */
3684         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3685                                          actions, false, error);
3686         if (!mcp_res->rix_flow) {
3687                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
3688                 return NULL;
3689         }
3690         return &mcp_res->hlist_ent;
3691 }
3692
3693 /**
3694  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3695  *
3696  * As mark_id is unique, if there's already a registered flow for the mark_id,
3697  * return by increasing the reference counter of the resource. Otherwise, create
3698  * the resource (mcp_res) and flow.
3699  *
3700  * Flow looks like,
3701  *   - If ingress port is ANY and reg_c[1] is mark_id,
3702  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3703  *
3704  * For default flow (zero mark_id), flow is like,
3705  *   - If ingress port is ANY,
3706  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3707  *
3708  * @param dev
3709  *   Pointer to Ethernet device.
3710  * @param mark_id
3711  *   ID of MARK action, zero means default flow for META.
3712  * @param[out] error
3713  *   Perform verbose error reporting if not NULL.
3714  *
3715  * @return
3716  *   Associated resource on success, NULL otherwise and rte_errno is set.
3717  */
3718 static struct mlx5_flow_mreg_copy_resource *
3719 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3720                           struct rte_flow_error *error)
3721 {
3722         struct mlx5_priv *priv = dev->data->dev_private;
3723         struct mlx5_hlist_entry *entry;
3724         struct mlx5_flow_cb_ctx ctx = {
3725                 .dev = dev,
3726                 .error = error,
3727         };
3728
3729         /* Check if already registered. */
3730         MLX5_ASSERT(priv->mreg_cp_tbl);
3731         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
3732         if (!entry)
3733                 return NULL;
3734         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
3735                             hlist_ent);
3736 }
3737
3738 void
3739 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
3740 {
3741         struct mlx5_flow_mreg_copy_resource *mcp_res =
3742                 container_of(entry, typeof(*mcp_res), hlist_ent);
3743         struct rte_eth_dev *dev = list->ctx;
3744         struct mlx5_priv *priv = dev->data->dev_private;
3745
3746         MLX5_ASSERT(mcp_res->rix_flow);
3747         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3748         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3749 }
3750
3751 /**
3752  * Release flow in RX_CP_TBL.
3753  *
3754  * @param dev
3755  *   Pointer to Ethernet device.
3756  * @flow
3757  *   Parent flow for wich copying is provided.
3758  */
3759 static void
3760 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3761                           struct rte_flow *flow)
3762 {
3763         struct mlx5_flow_mreg_copy_resource *mcp_res;
3764         struct mlx5_priv *priv = dev->data->dev_private;
3765
3766         if (!flow->rix_mreg_copy)
3767                 return;
3768         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3769                                  flow->rix_mreg_copy);
3770         if (!mcp_res || !priv->mreg_cp_tbl)
3771                 return;
3772         MLX5_ASSERT(mcp_res->rix_flow);
3773         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3774         flow->rix_mreg_copy = 0;
3775 }
3776
3777 /**
3778  * Remove the default copy action from RX_CP_TBL.
3779  *
3780  * This functions is called in the mlx5_dev_start(). No thread safe
3781  * is guaranteed.
3782  *
3783  * @param dev
3784  *   Pointer to Ethernet device.
3785  */
3786 static void
3787 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3788 {
3789         struct mlx5_hlist_entry *entry;
3790         struct mlx5_priv *priv = dev->data->dev_private;
3791
3792         /* Check if default flow is registered. */
3793         if (!priv->mreg_cp_tbl)
3794                 return;
3795         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
3796                                   MLX5_DEFAULT_COPY_ID, NULL);
3797         if (!entry)
3798                 return;
3799         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
3800 }
3801
3802 /**
3803  * Add the default copy action in in RX_CP_TBL.
3804  *
3805  * This functions is called in the mlx5_dev_start(). No thread safe
3806  * is guaranteed.
3807  *
3808  * @param dev
3809  *   Pointer to Ethernet device.
3810  * @param[out] error
3811  *   Perform verbose error reporting if not NULL.
3812  *
3813  * @return
3814  *   0 for success, negative value otherwise and rte_errno is set.
3815  */
3816 static int
3817 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3818                                   struct rte_flow_error *error)
3819 {
3820         struct mlx5_priv *priv = dev->data->dev_private;
3821         struct mlx5_flow_mreg_copy_resource *mcp_res;
3822
3823         /* Check whether extensive metadata feature is engaged. */
3824         if (!priv->config.dv_flow_en ||
3825             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3826             !mlx5_flow_ext_mreg_supported(dev) ||
3827             !priv->sh->dv_regc0_mask)
3828                 return 0;
3829         /*
3830          * Add default mreg copy flow may be called multiple time, but
3831          * only be called once in stop. Avoid register it twice.
3832          */
3833         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
3834                 return 0;
3835         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3836         if (!mcp_res)
3837                 return -rte_errno;
3838         return 0;
3839 }
3840
3841 /**
3842  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3843  *
3844  * All the flow having Q/RSS action should be split by
3845  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3846  * performs the following,
3847  *   - CQE->flow_tag := reg_c[1] (MARK)
3848  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3849  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3850  * but there should be a flow per each MARK ID set by MARK action.
3851  *
3852  * For the aforementioned reason, if there's a MARK action in flow's action
3853  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3854  * the MARK ID to CQE's flow_tag like,
3855  *   - If reg_c[1] is mark_id,
3856  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3857  *
3858  * For SET_META action which stores value in reg_c[0], as the destination is
3859  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3860  * MARK ID means the default flow. The default flow looks like,
3861  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3862  *
3863  * @param dev
3864  *   Pointer to Ethernet device.
3865  * @param flow
3866  *   Pointer to flow structure.
3867  * @param[in] actions
3868  *   Pointer to the list of actions.
3869  * @param[out] error
3870  *   Perform verbose error reporting if not NULL.
3871  *
3872  * @return
3873  *   0 on success, negative value otherwise and rte_errno is set.
3874  */
3875 static int
3876 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3877                             struct rte_flow *flow,
3878                             const struct rte_flow_action *actions,
3879                             struct rte_flow_error *error)
3880 {
3881         struct mlx5_priv *priv = dev->data->dev_private;
3882         struct mlx5_dev_config *config = &priv->config;
3883         struct mlx5_flow_mreg_copy_resource *mcp_res;
3884         const struct rte_flow_action_mark *mark;
3885
3886         /* Check whether extensive metadata feature is engaged. */
3887         if (!config->dv_flow_en ||
3888             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3889             !mlx5_flow_ext_mreg_supported(dev) ||
3890             !priv->sh->dv_regc0_mask)
3891                 return 0;
3892         /* Find MARK action. */
3893         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3894                 switch (actions->type) {
3895                 case RTE_FLOW_ACTION_TYPE_FLAG:
3896                         mcp_res = flow_mreg_add_copy_action
3897                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3898                         if (!mcp_res)
3899                                 return -rte_errno;
3900                         flow->rix_mreg_copy = mcp_res->idx;
3901                         return 0;
3902                 case RTE_FLOW_ACTION_TYPE_MARK:
3903                         mark = (const struct rte_flow_action_mark *)
3904                                 actions->conf;
3905                         mcp_res =
3906                                 flow_mreg_add_copy_action(dev, mark->id, error);
3907                         if (!mcp_res)
3908                                 return -rte_errno;
3909                         flow->rix_mreg_copy = mcp_res->idx;
3910                         return 0;
3911                 default:
3912                         break;
3913                 }
3914         }
3915         return 0;
3916 }
3917
3918 #define MLX5_MAX_SPLIT_ACTIONS 24
3919 #define MLX5_MAX_SPLIT_ITEMS 24
3920
3921 /**
3922  * Split the hairpin flow.
3923  * Since HW can't support encap and push-vlan on Rx, we move these
3924  * actions to Tx.
3925  * If the count action is after the encap then we also
3926  * move the count action. in this case the count will also measure
3927  * the outer bytes.
3928  *
3929  * @param dev
3930  *   Pointer to Ethernet device.
3931  * @param[in] actions
3932  *   Associated actions (list terminated by the END action).
3933  * @param[out] actions_rx
3934  *   Rx flow actions.
3935  * @param[out] actions_tx
3936  *   Tx flow actions..
3937  * @param[out] pattern_tx
3938  *   The pattern items for the Tx flow.
3939  * @param[out] flow_id
3940  *   The flow ID connected to this flow.
3941  *
3942  * @return
3943  *   0 on success.
3944  */
3945 static int
3946 flow_hairpin_split(struct rte_eth_dev *dev,
3947                    const struct rte_flow_action actions[],
3948                    struct rte_flow_action actions_rx[],
3949                    struct rte_flow_action actions_tx[],
3950                    struct rte_flow_item pattern_tx[],
3951                    uint32_t flow_id)
3952 {
3953         const struct rte_flow_action_raw_encap *raw_encap;
3954         const struct rte_flow_action_raw_decap *raw_decap;
3955         struct mlx5_rte_flow_action_set_tag *set_tag;
3956         struct rte_flow_action *tag_action;
3957         struct mlx5_rte_flow_item_tag *tag_item;
3958         struct rte_flow_item *item;
3959         char *addr;
3960         int encap = 0;
3961
3962         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3963                 switch (actions->type) {
3964                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3965                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3966                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3967                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3968                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3969                         rte_memcpy(actions_tx, actions,
3970                                sizeof(struct rte_flow_action));
3971                         actions_tx++;
3972                         break;
3973                 case RTE_FLOW_ACTION_TYPE_COUNT:
3974                         if (encap) {
3975                                 rte_memcpy(actions_tx, actions,
3976                                            sizeof(struct rte_flow_action));
3977                                 actions_tx++;
3978                         } else {
3979                                 rte_memcpy(actions_rx, actions,
3980                                            sizeof(struct rte_flow_action));
3981                                 actions_rx++;
3982                         }
3983                         break;
3984                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3985                         raw_encap = actions->conf;
3986                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
3987                                 memcpy(actions_tx, actions,
3988                                        sizeof(struct rte_flow_action));
3989                                 actions_tx++;
3990                                 encap = 1;
3991                         } else {
3992                                 rte_memcpy(actions_rx, actions,
3993                                            sizeof(struct rte_flow_action));
3994                                 actions_rx++;
3995                         }
3996                         break;
3997                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3998                         raw_decap = actions->conf;
3999                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4000                                 memcpy(actions_tx, actions,
4001                                        sizeof(struct rte_flow_action));
4002                                 actions_tx++;
4003                         } else {
4004                                 rte_memcpy(actions_rx, actions,
4005                                            sizeof(struct rte_flow_action));
4006                                 actions_rx++;
4007                         }
4008                         break;
4009                 default:
4010                         rte_memcpy(actions_rx, actions,
4011                                    sizeof(struct rte_flow_action));
4012                         actions_rx++;
4013                         break;
4014                 }
4015         }
4016         /* Add set meta action and end action for the Rx flow. */
4017         tag_action = actions_rx;
4018         tag_action->type = (enum rte_flow_action_type)
4019                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4020         actions_rx++;
4021         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4022         actions_rx++;
4023         set_tag = (void *)actions_rx;
4024         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4025         MLX5_ASSERT(set_tag->id > REG_NON);
4026         set_tag->data = flow_id;
4027         tag_action->conf = set_tag;
4028         /* Create Tx item list. */
4029         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4030         addr = (void *)&pattern_tx[2];
4031         item = pattern_tx;
4032         item->type = (enum rte_flow_item_type)
4033                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4034         tag_item = (void *)addr;
4035         tag_item->data = flow_id;
4036         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4037         MLX5_ASSERT(set_tag->id > REG_NON);
4038         item->spec = tag_item;
4039         addr += sizeof(struct mlx5_rte_flow_item_tag);
4040         tag_item = (void *)addr;
4041         tag_item->data = UINT32_MAX;
4042         tag_item->id = UINT16_MAX;
4043         item->mask = tag_item;
4044         item->last = NULL;
4045         item++;
4046         item->type = RTE_FLOW_ITEM_TYPE_END;
4047         return 0;
4048 }
4049
4050 /**
4051  * The last stage of splitting chain, just creates the subflow
4052  * without any modification.
4053  *
4054  * @param[in] dev
4055  *   Pointer to Ethernet device.
4056  * @param[in] flow
4057  *   Parent flow structure pointer.
4058  * @param[in, out] sub_flow
4059  *   Pointer to return the created subflow, may be NULL.
4060  * @param[in] attr
4061  *   Flow rule attributes.
4062  * @param[in] items
4063  *   Pattern specification (list terminated by the END pattern item).
4064  * @param[in] actions
4065  *   Associated actions (list terminated by the END action).
4066  * @param[in] flow_split_info
4067  *   Pointer to flow split info structure.
4068  * @param[out] error
4069  *   Perform verbose error reporting if not NULL.
4070  * @return
4071  *   0 on success, negative value otherwise
4072  */
4073 static int
4074 flow_create_split_inner(struct rte_eth_dev *dev,
4075                         struct rte_flow *flow,
4076                         struct mlx5_flow **sub_flow,
4077                         const struct rte_flow_attr *attr,
4078                         const struct rte_flow_item items[],
4079                         const struct rte_flow_action actions[],
4080                         struct mlx5_flow_split_info *flow_split_info,
4081                         struct rte_flow_error *error)
4082 {
4083         struct mlx5_flow *dev_flow;
4084
4085         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4086                                     flow_split_info->flow_idx, error);
4087         if (!dev_flow)
4088                 return -rte_errno;
4089         dev_flow->flow = flow;
4090         dev_flow->external = flow_split_info->external;
4091         dev_flow->skip_scale = flow_split_info->skip_scale;
4092         /* Subflow object was created, we must include one in the list. */
4093         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4094                       dev_flow->handle, next);
4095         /*
4096          * If dev_flow is as one of the suffix flow, some actions in suffix
4097          * flow may need some user defined item layer flags, and pass the
4098          * Metadate rxq mark flag to suffix flow as well.
4099          */
4100         if (flow_split_info->prefix_layers)
4101                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4102         if (flow_split_info->prefix_mark)
4103                 dev_flow->handle->mark = 1;
4104         if (sub_flow)
4105                 *sub_flow = dev_flow;
4106         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4107 }
4108
4109 /**
4110  * Split the meter flow.
4111  *
4112  * As meter flow will split to three sub flow, other than meter
4113  * action, the other actions make sense to only meter accepts
4114  * the packet. If it need to be dropped, no other additional
4115  * actions should be take.
4116  *
4117  * One kind of special action which decapsulates the L3 tunnel
4118  * header will be in the prefix sub flow, as not to take the
4119  * L3 tunnel header into account.
4120  *
4121  * @param dev
4122  *   Pointer to Ethernet device.
4123  * @param[in] items
4124  *   Pattern specification (list terminated by the END pattern item).
4125  * @param[out] sfx_items
4126  *   Suffix flow match items (list terminated by the END pattern item).
4127  * @param[in] actions
4128  *   Associated actions (list terminated by the END action).
4129  * @param[out] actions_sfx
4130  *   Suffix flow actions.
4131  * @param[out] actions_pre
4132  *   Prefix flow actions.
4133  * @param[out] pattern_sfx
4134  *   The pattern items for the suffix flow.
4135  * @param[out] tag_sfx
4136  *   Pointer to suffix flow tag.
4137  *
4138  * @return
4139  *   0 on success.
4140  */
4141 static int
4142 flow_meter_split_prep(struct rte_eth_dev *dev,
4143                  const struct rte_flow_item items[],
4144                  struct rte_flow_item sfx_items[],
4145                  const struct rte_flow_action actions[],
4146                  struct rte_flow_action actions_sfx[],
4147                  struct rte_flow_action actions_pre[])
4148 {
4149         struct mlx5_priv *priv = dev->data->dev_private;
4150         struct rte_flow_action *tag_action = NULL;
4151         struct rte_flow_item *tag_item;
4152         struct mlx5_rte_flow_action_set_tag *set_tag;
4153         struct rte_flow_error error;
4154         const struct rte_flow_action_raw_encap *raw_encap;
4155         const struct rte_flow_action_raw_decap *raw_decap;
4156         struct mlx5_rte_flow_item_tag *tag_spec;
4157         struct mlx5_rte_flow_item_tag *tag_mask;
4158         uint32_t tag_id = 0;
4159         bool copy_vlan = false;
4160
4161         /* Prepare the actions for prefix and suffix flow. */
4162         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4163                 struct rte_flow_action **action_cur = NULL;
4164
4165                 switch (actions->type) {
4166                 case RTE_FLOW_ACTION_TYPE_METER:
4167                         /* Add the extra tag action first. */
4168                         tag_action = actions_pre;
4169                         tag_action->type = (enum rte_flow_action_type)
4170                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4171                         actions_pre++;
4172                         action_cur = &actions_pre;
4173                         break;
4174                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4175                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4176                         action_cur = &actions_pre;
4177                         break;
4178                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4179                         raw_encap = actions->conf;
4180                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4181                                 action_cur = &actions_pre;
4182                         break;
4183                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4184                         raw_decap = actions->conf;
4185                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4186                                 action_cur = &actions_pre;
4187                         break;
4188                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4189                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4190                         copy_vlan = true;
4191                         break;
4192                 default:
4193                         break;
4194                 }
4195                 if (!action_cur)
4196                         action_cur = &actions_sfx;
4197                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4198                 (*action_cur)++;
4199         }
4200         /* Add end action to the actions. */
4201         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4202         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4203         actions_pre++;
4204         /* Set the tag. */
4205         set_tag = (void *)actions_pre;
4206         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4207         mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4208                           &tag_id);
4209         if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
4210                 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
4211                         dev->data->port_id);
4212                 mlx5_ipool_free(priv->sh->ipool
4213                                 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
4214                 return 0;
4215         } else if (!tag_id) {
4216                 return 0;
4217         }
4218         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4219         assert(tag_action);
4220         tag_action->conf = set_tag;
4221         /* Prepare the suffix subflow items. */
4222         tag_item = sfx_items++;
4223         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4224                 int item_type = items->type;
4225
4226                 switch (item_type) {
4227                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4228                         memcpy(sfx_items, items, sizeof(*sfx_items));
4229                         sfx_items++;
4230                         break;
4231                 case RTE_FLOW_ITEM_TYPE_VLAN:
4232                         if (copy_vlan) {
4233                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4234                                 /*
4235                                  * Convert to internal match item, it is used
4236                                  * for vlan push and set vid.
4237                                  */
4238                                 sfx_items->type = (enum rte_flow_item_type)
4239                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4240                                 sfx_items++;
4241                         }
4242                         break;
4243                 default:
4244                         break;
4245                 }
4246         }
4247         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4248         sfx_items++;
4249         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4250         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4251         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4252         tag_mask = tag_spec + 1;
4253         tag_mask->data = 0xffffff00;
4254         tag_item->type = (enum rte_flow_item_type)
4255                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4256         tag_item->spec = tag_spec;
4257         tag_item->last = NULL;
4258         tag_item->mask = tag_mask;
4259         return tag_id;
4260 }
4261
4262 /**
4263  * Split action list having QUEUE/RSS for metadata register copy.
4264  *
4265  * Once Q/RSS action is detected in user's action list, the flow action
4266  * should be split in order to copy metadata registers, which will happen in
4267  * RX_CP_TBL like,
4268  *   - CQE->flow_tag := reg_c[1] (MARK)
4269  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4270  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4271  * This is because the last action of each flow must be a terminal action
4272  * (QUEUE, RSS or DROP).
4273  *
4274  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4275  * stored and kept in the mlx5_flow structure per each sub_flow.
4276  *
4277  * The Q/RSS action is replaced with,
4278  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4279  * And the following JUMP action is added at the end,
4280  *   - JUMP, to RX_CP_TBL.
4281  *
4282  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4283  * flow_create_split_metadata() routine. The flow will look like,
4284  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4285  *
4286  * @param dev
4287  *   Pointer to Ethernet device.
4288  * @param[out] split_actions
4289  *   Pointer to store split actions to jump to CP_TBL.
4290  * @param[in] actions
4291  *   Pointer to the list of original flow actions.
4292  * @param[in] qrss
4293  *   Pointer to the Q/RSS action.
4294  * @param[in] actions_n
4295  *   Number of original actions.
4296  * @param[out] error
4297  *   Perform verbose error reporting if not NULL.
4298  *
4299  * @return
4300  *   non-zero unique flow_id on success, otherwise 0 and
4301  *   error/rte_error are set.
4302  */
4303 static uint32_t
4304 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4305                           struct rte_flow_action *split_actions,
4306                           const struct rte_flow_action *actions,
4307                           const struct rte_flow_action *qrss,
4308                           int actions_n, struct rte_flow_error *error)
4309 {
4310         struct mlx5_priv *priv = dev->data->dev_private;
4311         struct mlx5_rte_flow_action_set_tag *set_tag;
4312         struct rte_flow_action_jump *jump;
4313         const int qrss_idx = qrss - actions;
4314         uint32_t flow_id = 0;
4315         int ret = 0;
4316
4317         /*
4318          * Given actions will be split
4319          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4320          * - Add jump to mreg CP_TBL.
4321          * As a result, there will be one more action.
4322          */
4323         ++actions_n;
4324         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4325         set_tag = (void *)(split_actions + actions_n);
4326         /*
4327          * If tag action is not set to void(it means we are not the meter
4328          * suffix flow), add the tag action. Since meter suffix flow already
4329          * has the tag added.
4330          */
4331         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4332                 /*
4333                  * Allocate the new subflow ID. This one is unique within
4334                  * device and not shared with representors. Otherwise,
4335                  * we would have to resolve multi-thread access synch
4336                  * issue. Each flow on the shared device is appended
4337                  * with source vport identifier, so the resulting
4338                  * flows will be unique in the shared (by master and
4339                  * representors) domain even if they have coinciding
4340                  * IDs.
4341                  */
4342                 mlx5_ipool_malloc(priv->sh->ipool
4343                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
4344                 if (!flow_id)
4345                         return rte_flow_error_set(error, ENOMEM,
4346                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4347                                                   NULL, "can't allocate id "
4348                                                   "for split Q/RSS subflow");
4349                 /* Internal SET_TAG action to set flow ID. */
4350                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4351                         .data = flow_id,
4352                 };
4353                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4354                 if (ret < 0)
4355                         return ret;
4356                 set_tag->id = ret;
4357                 /* Construct new actions array. */
4358                 /* Replace QUEUE/RSS action. */
4359                 split_actions[qrss_idx] = (struct rte_flow_action){
4360                         .type = (enum rte_flow_action_type)
4361                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4362                         .conf = set_tag,
4363                 };
4364         }
4365         /* JUMP action to jump to mreg copy table (CP_TBL). */
4366         jump = (void *)(set_tag + 1);
4367         *jump = (struct rte_flow_action_jump){
4368                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4369         };
4370         split_actions[actions_n - 2] = (struct rte_flow_action){
4371                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4372                 .conf = jump,
4373         };
4374         split_actions[actions_n - 1] = (struct rte_flow_action){
4375                 .type = RTE_FLOW_ACTION_TYPE_END,
4376         };
4377         return flow_id;
4378 }
4379
4380 /**
4381  * Extend the given action list for Tx metadata copy.
4382  *
4383  * Copy the given action list to the ext_actions and add flow metadata register
4384  * copy action in order to copy reg_a set by WQE to reg_c[0].
4385  *
4386  * @param[out] ext_actions
4387  *   Pointer to the extended action list.
4388  * @param[in] actions
4389  *   Pointer to the list of actions.
4390  * @param[in] actions_n
4391  *   Number of actions in the list.
4392  * @param[out] error
4393  *   Perform verbose error reporting if not NULL.
4394  * @param[in] encap_idx
4395  *   The encap action inndex.
4396  *
4397  * @return
4398  *   0 on success, negative value otherwise
4399  */
4400 static int
4401 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4402                        struct rte_flow_action *ext_actions,
4403                        const struct rte_flow_action *actions,
4404                        int actions_n, struct rte_flow_error *error,
4405                        int encap_idx)
4406 {
4407         struct mlx5_flow_action_copy_mreg *cp_mreg =
4408                 (struct mlx5_flow_action_copy_mreg *)
4409                         (ext_actions + actions_n + 1);
4410         int ret;
4411
4412         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4413         if (ret < 0)
4414                 return ret;
4415         cp_mreg->dst = ret;
4416         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4417         if (ret < 0)
4418                 return ret;
4419         cp_mreg->src = ret;
4420         if (encap_idx != 0)
4421                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4422         if (encap_idx == actions_n - 1) {
4423                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4424                         .type = (enum rte_flow_action_type)
4425                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4426                         .conf = cp_mreg,
4427                 };
4428                 ext_actions[actions_n] = (struct rte_flow_action){
4429                         .type = RTE_FLOW_ACTION_TYPE_END,
4430                 };
4431         } else {
4432                 ext_actions[encap_idx] = (struct rte_flow_action){
4433                         .type = (enum rte_flow_action_type)
4434                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4435                         .conf = cp_mreg,
4436                 };
4437                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4438                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4439         }
4440         return 0;
4441 }
4442
4443 /**
4444  * Check the match action from the action list.
4445  *
4446  * @param[in] actions
4447  *   Pointer to the list of actions.
4448  * @param[in] attr
4449  *   Flow rule attributes.
4450  * @param[in] action
4451  *   The action to be check if exist.
4452  * @param[out] match_action_pos
4453  *   Pointer to the position of the matched action if exists, otherwise is -1.
4454  * @param[out] qrss_action_pos
4455  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4456  *
4457  * @return
4458  *   > 0 the total number of actions.
4459  *   0 if not found match action in action list.
4460  */
4461 static int
4462 flow_check_match_action(const struct rte_flow_action actions[],
4463                         const struct rte_flow_attr *attr,
4464                         enum rte_flow_action_type action,
4465                         int *match_action_pos, int *qrss_action_pos)
4466 {
4467         const struct rte_flow_action_sample *sample;
4468         int actions_n = 0;
4469         int jump_flag = 0;
4470         uint32_t ratio = 0;
4471         int sub_type = 0;
4472         int flag = 0;
4473
4474         *match_action_pos = -1;
4475         *qrss_action_pos = -1;
4476         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4477                 if (actions->type == action) {
4478                         flag = 1;
4479                         *match_action_pos = actions_n;
4480                 }
4481                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4482                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4483                         *qrss_action_pos = actions_n;
4484                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4485                         jump_flag = 1;
4486                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4487                         sample = actions->conf;
4488                         ratio = sample->ratio;
4489                         sub_type = ((const struct rte_flow_action *)
4490                                         (sample->actions))->type;
4491                 }
4492                 actions_n++;
4493         }
4494         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4495                 if (ratio == 1) {
4496                         /* JUMP Action not support for Mirroring;
4497                          * Mirroring support multi-destination;
4498                          */
4499                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4500                                 flag = 0;
4501                 }
4502         }
4503         /* Count RTE_FLOW_ACTION_TYPE_END. */
4504         return flag ? actions_n + 1 : 0;
4505 }
4506
4507 #define SAMPLE_SUFFIX_ITEM 2
4508
4509 /**
4510  * Split the sample flow.
4511  *
4512  * As sample flow will split to two sub flow, sample flow with
4513  * sample action, the other actions will move to new suffix flow.
4514  *
4515  * Also add unique tag id with tag action in the sample flow,
4516  * the same tag id will be as match in the suffix flow.
4517  *
4518  * @param dev
4519  *   Pointer to Ethernet device.
4520  * @param[in] fdb_tx
4521  *   FDB egress flow flag.
4522  * @param[out] sfx_items
4523  *   Suffix flow match items (list terminated by the END pattern item).
4524  * @param[in] actions
4525  *   Associated actions (list terminated by the END action).
4526  * @param[out] actions_sfx
4527  *   Suffix flow actions.
4528  * @param[out] actions_pre
4529  *   Prefix flow actions.
4530  * @param[in] actions_n
4531  *  The total number of actions.
4532  * @param[in] sample_action_pos
4533  *   The sample action position.
4534  * @param[in] qrss_action_pos
4535  *   The Queue/RSS action position.
4536  * @param[out] error
4537  *   Perform verbose error reporting if not NULL.
4538  *
4539  * @return
4540  *   0 on success, or unique flow_id, a negative errno value
4541  *   otherwise and rte_errno is set.
4542  */
4543 static int
4544 flow_sample_split_prep(struct rte_eth_dev *dev,
4545                        uint32_t fdb_tx,
4546                        struct rte_flow_item sfx_items[],
4547                        const struct rte_flow_action actions[],
4548                        struct rte_flow_action actions_sfx[],
4549                        struct rte_flow_action actions_pre[],
4550                        int actions_n,
4551                        int sample_action_pos,
4552                        int qrss_action_pos,
4553                        struct rte_flow_error *error)
4554 {
4555         struct mlx5_priv *priv = dev->data->dev_private;
4556         struct mlx5_rte_flow_action_set_tag *set_tag;
4557         struct mlx5_rte_flow_item_tag *tag_spec;
4558         struct mlx5_rte_flow_item_tag *tag_mask;
4559         uint32_t tag_id = 0;
4560         int index;
4561         int ret;
4562
4563         if (sample_action_pos < 0)
4564                 return rte_flow_error_set(error, EINVAL,
4565                                           RTE_FLOW_ERROR_TYPE_ACTION,
4566                                           NULL, "invalid position of sample "
4567                                           "action in list");
4568         if (!fdb_tx) {
4569                 /* Prepare the prefix tag action. */
4570                 set_tag = (void *)(actions_pre + actions_n + 1);
4571                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4572                 if (ret < 0)
4573                         return ret;
4574                 set_tag->id = ret;
4575                 mlx5_ipool_malloc(priv->sh->ipool
4576                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
4577                 set_tag->data = tag_id;
4578                 /* Prepare the suffix subflow items. */
4579                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4580                 tag_spec->data = tag_id;
4581                 tag_spec->id = set_tag->id;
4582                 tag_mask = tag_spec + 1;
4583                 tag_mask->data = UINT32_MAX;
4584                 sfx_items[0] = (struct rte_flow_item){
4585                         .type = (enum rte_flow_item_type)
4586                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4587                         .spec = tag_spec,
4588                         .last = NULL,
4589                         .mask = tag_mask,
4590                 };
4591                 sfx_items[1] = (struct rte_flow_item){
4592                         .type = (enum rte_flow_item_type)
4593                                 RTE_FLOW_ITEM_TYPE_END,
4594                 };
4595         }
4596         /* Prepare the actions for prefix and suffix flow. */
4597         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4598                 index = qrss_action_pos;
4599                 /* Put the preceding the Queue/RSS action into prefix flow. */
4600                 if (index != 0)
4601                         memcpy(actions_pre, actions,
4602                                sizeof(struct rte_flow_action) * index);
4603                 /* Put others preceding the sample action into prefix flow. */
4604                 if (sample_action_pos > index + 1)
4605                         memcpy(actions_pre + index, actions + index + 1,
4606                                sizeof(struct rte_flow_action) *
4607                                (sample_action_pos - index - 1));
4608                 index = sample_action_pos - 1;
4609                 /* Put Queue/RSS action into Suffix flow. */
4610                 memcpy(actions_sfx, actions + qrss_action_pos,
4611                        sizeof(struct rte_flow_action));
4612                 actions_sfx++;
4613         } else {
4614                 index = sample_action_pos;
4615                 if (index != 0)
4616                         memcpy(actions_pre, actions,
4617                                sizeof(struct rte_flow_action) * index);
4618         }
4619         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4620         if (!fdb_tx) {
4621                 actions_pre[index++] =
4622                         (struct rte_flow_action){
4623                         .type = (enum rte_flow_action_type)
4624                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4625                         .conf = set_tag,
4626                 };
4627         }
4628         memcpy(actions_pre + index, actions + sample_action_pos,
4629                sizeof(struct rte_flow_action));
4630         index += 1;
4631         actions_pre[index] = (struct rte_flow_action){
4632                 .type = (enum rte_flow_action_type)
4633                         RTE_FLOW_ACTION_TYPE_END,
4634         };
4635         /* Put the actions after sample into Suffix flow. */
4636         memcpy(actions_sfx, actions + sample_action_pos + 1,
4637                sizeof(struct rte_flow_action) *
4638                (actions_n - sample_action_pos - 1));
4639         return tag_id;
4640 }
4641
4642 /**
4643  * The splitting for metadata feature.
4644  *
4645  * - Q/RSS action on NIC Rx should be split in order to pass by
4646  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4647  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4648  *
4649  * - All the actions on NIC Tx should have a mreg copy action to
4650  *   copy reg_a from WQE to reg_c[0].
4651  *
4652  * @param dev
4653  *   Pointer to Ethernet device.
4654  * @param[in] flow
4655  *   Parent flow structure pointer.
4656  * @param[in] attr
4657  *   Flow rule attributes.
4658  * @param[in] items
4659  *   Pattern specification (list terminated by the END pattern item).
4660  * @param[in] actions
4661  *   Associated actions (list terminated by the END action).
4662  * @param[in] flow_split_info
4663  *   Pointer to flow split info structure.
4664  * @param[out] error
4665  *   Perform verbose error reporting if not NULL.
4666  * @return
4667  *   0 on success, negative value otherwise
4668  */
4669 static int
4670 flow_create_split_metadata(struct rte_eth_dev *dev,
4671                            struct rte_flow *flow,
4672                            const struct rte_flow_attr *attr,
4673                            const struct rte_flow_item items[],
4674                            const struct rte_flow_action actions[],
4675                            struct mlx5_flow_split_info *flow_split_info,
4676                            struct rte_flow_error *error)
4677 {
4678         struct mlx5_priv *priv = dev->data->dev_private;
4679         struct mlx5_dev_config *config = &priv->config;
4680         const struct rte_flow_action *qrss = NULL;
4681         struct rte_flow_action *ext_actions = NULL;
4682         struct mlx5_flow *dev_flow = NULL;
4683         uint32_t qrss_id = 0;
4684         int mtr_sfx = 0;
4685         size_t act_size;
4686         int actions_n;
4687         int encap_idx;
4688         int ret;
4689
4690         /* Check whether extensive metadata feature is engaged. */
4691         if (!config->dv_flow_en ||
4692             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4693             !mlx5_flow_ext_mreg_supported(dev))
4694                 return flow_create_split_inner(dev, flow, NULL, attr, items,
4695                                                actions, flow_split_info, error);
4696         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4697                                                            &encap_idx);
4698         if (qrss) {
4699                 /* Exclude hairpin flows from splitting. */
4700                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4701                         const struct rte_flow_action_queue *queue;
4702
4703                         queue = qrss->conf;
4704                         if (mlx5_rxq_get_type(dev, queue->index) ==
4705                             MLX5_RXQ_TYPE_HAIRPIN)
4706                                 qrss = NULL;
4707                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4708                         const struct rte_flow_action_rss *rss;
4709
4710                         rss = qrss->conf;
4711                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4712                             MLX5_RXQ_TYPE_HAIRPIN)
4713                                 qrss = NULL;
4714                 }
4715         }
4716         if (qrss) {
4717                 /* Check if it is in meter suffix table. */
4718                 mtr_sfx = attr->group == (attr->transfer ?
4719                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4720                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4721                 /*
4722                  * Q/RSS action on NIC Rx should be split in order to pass by
4723                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4724                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4725                  */
4726                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4727                            sizeof(struct rte_flow_action_set_tag) +
4728                            sizeof(struct rte_flow_action_jump);
4729                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4730                                           SOCKET_ID_ANY);
4731                 if (!ext_actions)
4732                         return rte_flow_error_set(error, ENOMEM,
4733                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4734                                                   NULL, "no memory to split "
4735                                                   "metadata flow");
4736                 /*
4737                  * If we are the suffix flow of meter, tag already exist.
4738                  * Set the tag action to void.
4739                  */
4740                 if (mtr_sfx)
4741                         ext_actions[qrss - actions].type =
4742                                                 RTE_FLOW_ACTION_TYPE_VOID;
4743                 else
4744                         ext_actions[qrss - actions].type =
4745                                                 (enum rte_flow_action_type)
4746                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4747                 /*
4748                  * Create the new actions list with removed Q/RSS action
4749                  * and appended set tag and jump to register copy table
4750                  * (RX_CP_TBL). We should preallocate unique tag ID here
4751                  * in advance, because it is needed for set tag action.
4752                  */
4753                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4754                                                     qrss, actions_n, error);
4755                 if (!mtr_sfx && !qrss_id) {
4756                         ret = -rte_errno;
4757                         goto exit;
4758                 }
4759         } else if (attr->egress && !attr->transfer) {
4760                 /*
4761                  * All the actions on NIC Tx should have a metadata register
4762                  * copy action to copy reg_a from WQE to reg_c[meta]
4763                  */
4764                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4765                            sizeof(struct mlx5_flow_action_copy_mreg);
4766                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4767                                           SOCKET_ID_ANY);
4768                 if (!ext_actions)
4769                         return rte_flow_error_set(error, ENOMEM,
4770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4771                                                   NULL, "no memory to split "
4772                                                   "metadata flow");
4773                 /* Create the action list appended with copy register. */
4774                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4775                                              actions_n, error, encap_idx);
4776                 if (ret < 0)
4777                         goto exit;
4778         }
4779         /* Add the unmodified original or prefix subflow. */
4780         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
4781                                       items, ext_actions ? ext_actions :
4782                                       actions, flow_split_info, error);
4783         if (ret < 0)
4784                 goto exit;
4785         MLX5_ASSERT(dev_flow);
4786         if (qrss) {
4787                 const struct rte_flow_attr q_attr = {
4788                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4789                         .ingress = 1,
4790                 };
4791                 /* Internal PMD action to set register. */
4792                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4793                         .data = qrss_id,
4794                         .id = REG_NON,
4795                 };
4796                 struct rte_flow_item q_items[] = {
4797                         {
4798                                 .type = (enum rte_flow_item_type)
4799                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4800                                 .spec = &q_tag_spec,
4801                                 .last = NULL,
4802                                 .mask = NULL,
4803                         },
4804                         {
4805                                 .type = RTE_FLOW_ITEM_TYPE_END,
4806                         },
4807                 };
4808                 struct rte_flow_action q_actions[] = {
4809                         {
4810                                 .type = qrss->type,
4811                                 .conf = qrss->conf,
4812                         },
4813                         {
4814                                 .type = RTE_FLOW_ACTION_TYPE_END,
4815                         },
4816                 };
4817                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4818
4819                 /*
4820                  * Configure the tag item only if there is no meter subflow.
4821                  * Since tag is already marked in the meter suffix subflow
4822                  * we can just use the meter suffix items as is.
4823                  */
4824                 if (qrss_id) {
4825                         /* Not meter subflow. */
4826                         MLX5_ASSERT(!mtr_sfx);
4827                         /*
4828                          * Put unique id in prefix flow due to it is destroyed
4829                          * after suffix flow and id will be freed after there
4830                          * is no actual flows with this id and identifier
4831                          * reallocation becomes possible (for example, for
4832                          * other flows in other threads).
4833                          */
4834                         dev_flow->handle->split_flow_id = qrss_id;
4835                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4836                                                    error);
4837                         if (ret < 0)
4838                                 goto exit;
4839                         q_tag_spec.id = ret;
4840                 }
4841                 dev_flow = NULL;
4842                 /* Add suffix subflow to execute Q/RSS. */
4843                 flow_split_info->prefix_layers = layers;
4844                 flow_split_info->prefix_mark = 0;
4845                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4846                                               &q_attr, mtr_sfx ? items :
4847                                               q_items, q_actions,
4848                                               flow_split_info, error);
4849                 if (ret < 0)
4850                         goto exit;
4851                 /* qrss ID should be freed if failed. */
4852                 qrss_id = 0;
4853                 MLX5_ASSERT(dev_flow);
4854         }
4855
4856 exit:
4857         /*
4858          * We do not destroy the partially created sub_flows in case of error.
4859          * These ones are included into parent flow list and will be destroyed
4860          * by flow_drv_destroy.
4861          */
4862         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
4863                         qrss_id);
4864         mlx5_free(ext_actions);
4865         return ret;
4866 }
4867
4868 /**
4869  * The splitting for meter feature.
4870  *
4871  * - The meter flow will be split to two flows as prefix and
4872  *   suffix flow. The packets make sense only it pass the prefix
4873  *   meter action.
4874  *
4875  * - Reg_C_5 is used for the packet to match betweend prefix and
4876  *   suffix flow.
4877  *
4878  * @param dev
4879  *   Pointer to Ethernet device.
4880  * @param[in] flow
4881  *   Parent flow structure pointer.
4882  * @param[in] attr
4883  *   Flow rule attributes.
4884  * @param[in] items
4885  *   Pattern specification (list terminated by the END pattern item).
4886  * @param[in] actions
4887  *   Associated actions (list terminated by the END action).
4888  * @param[in] flow_split_info
4889  *   Pointer to flow split info structure.
4890  * @param[out] error
4891  *   Perform verbose error reporting if not NULL.
4892  * @return
4893  *   0 on success, negative value otherwise
4894  */
4895 static int
4896 flow_create_split_meter(struct rte_eth_dev *dev,
4897                         struct rte_flow *flow,
4898                         const struct rte_flow_attr *attr,
4899                         const struct rte_flow_item items[],
4900                         const struct rte_flow_action actions[],
4901                         struct mlx5_flow_split_info *flow_split_info,
4902                         struct rte_flow_error *error)
4903 {
4904         struct mlx5_priv *priv = dev->data->dev_private;
4905         struct rte_flow_action *sfx_actions = NULL;
4906         struct rte_flow_action *pre_actions = NULL;
4907         struct rte_flow_item *sfx_items = NULL;
4908         struct mlx5_flow *dev_flow = NULL;
4909         struct rte_flow_attr sfx_attr = *attr;
4910         uint32_t mtr = 0;
4911         uint32_t mtr_tag_id = 0;
4912         size_t act_size;
4913         size_t item_size;
4914         int actions_n = 0;
4915         int ret;
4916
4917         if (priv->mtr_en)
4918                 actions_n = flow_check_meter_action(actions, &mtr);
4919         if (mtr) {
4920                 /* The five prefix actions: meter, decap, encap, tag, end. */
4921                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4922                            sizeof(struct mlx5_rte_flow_action_set_tag);
4923                 /* tag, vlan, port id, end. */
4924 #define METER_SUFFIX_ITEM 4
4925                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4926                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4927                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4928                                           0, SOCKET_ID_ANY);
4929                 if (!sfx_actions)
4930                         return rte_flow_error_set(error, ENOMEM,
4931                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4932                                                   NULL, "no memory to split "
4933                                                   "meter flow");
4934                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4935                              act_size);
4936                 pre_actions = sfx_actions + actions_n;
4937                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4938                                                    actions, sfx_actions,
4939                                                    pre_actions);
4940                 if (!mtr_tag_id) {
4941                         ret = -rte_errno;
4942                         goto exit;
4943                 }
4944                 /* Add the prefix subflow. */
4945                 flow_split_info->prefix_mark = 0;
4946                 ret = flow_create_split_inner(dev, flow, &dev_flow,
4947                                               attr, items, pre_actions,
4948                                               flow_split_info, error);
4949                 if (ret) {
4950                         ret = -rte_errno;
4951                         goto exit;
4952                 }
4953                 dev_flow->handle->split_flow_id = mtr_tag_id;
4954                 /* Setting the sfx group atrr. */
4955                 sfx_attr.group = sfx_attr.transfer ?
4956                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4957                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4958                 flow_split_info->prefix_layers =
4959                                 flow_get_prefix_layer_flags(dev_flow);
4960                 flow_split_info->prefix_mark = dev_flow->handle->mark;
4961         }
4962         /* Add the prefix subflow. */
4963         ret = flow_create_split_metadata(dev, flow,
4964                                          &sfx_attr, sfx_items ?
4965                                          sfx_items : items,
4966                                          sfx_actions ? sfx_actions : actions,
4967                                          flow_split_info, error);
4968 exit:
4969         if (sfx_actions)
4970                 mlx5_free(sfx_actions);
4971         return ret;
4972 }
4973
4974 /**
4975  * The splitting for sample feature.
4976  *
4977  * Once Sample action is detected in the action list, the flow actions should
4978  * be split into prefix sub flow and suffix sub flow.
4979  *
4980  * The original items remain in the prefix sub flow, all actions preceding the
4981  * sample action and the sample action itself will be copied to the prefix
4982  * sub flow, the actions following the sample action will be copied to the
4983  * suffix sub flow, Queue action always be located in the suffix sub flow.
4984  *
4985  * In order to make the packet from prefix sub flow matches with suffix sub
4986  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
4987  * flow uses tag item with the unique flow id.
4988  *
4989  * @param dev
4990  *   Pointer to Ethernet device.
4991  * @param[in] flow
4992  *   Parent flow structure pointer.
4993  * @param[in] attr
4994  *   Flow rule attributes.
4995  * @param[in] items
4996  *   Pattern specification (list terminated by the END pattern item).
4997  * @param[in] actions
4998  *   Associated actions (list terminated by the END action).
4999  * @param[in] flow_split_info
5000  *   Pointer to flow split info structure.
5001  * @param[out] error
5002  *   Perform verbose error reporting if not NULL.
5003  * @return
5004  *   0 on success, negative value otherwise
5005  */
5006 static int
5007 flow_create_split_sample(struct rte_eth_dev *dev,
5008                          struct rte_flow *flow,
5009                          const struct rte_flow_attr *attr,
5010                          const struct rte_flow_item items[],
5011                          const struct rte_flow_action actions[],
5012                          struct mlx5_flow_split_info *flow_split_info,
5013                          struct rte_flow_error *error)
5014 {
5015         struct mlx5_priv *priv = dev->data->dev_private;
5016         struct rte_flow_action *sfx_actions = NULL;
5017         struct rte_flow_action *pre_actions = NULL;
5018         struct rte_flow_item *sfx_items = NULL;
5019         struct mlx5_flow *dev_flow = NULL;
5020         struct rte_flow_attr sfx_attr = *attr;
5021 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5022         struct mlx5_flow_dv_sample_resource *sample_res;
5023         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5024         struct mlx5_flow_tbl_resource *sfx_tbl;
5025         union mlx5_flow_tbl_key sfx_table_key;
5026 #endif
5027         size_t act_size;
5028         size_t item_size;
5029         uint32_t fdb_tx = 0;
5030         int32_t tag_id = 0;
5031         int actions_n = 0;
5032         int sample_action_pos;
5033         int qrss_action_pos;
5034         int ret = 0;
5035
5036         if (priv->sampler_en)
5037                 actions_n = flow_check_match_action(actions, attr,
5038                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5039                                         &sample_action_pos, &qrss_action_pos);
5040         if (actions_n) {
5041                 /* The prefix actions must includes sample, tag, end. */
5042                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5043                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5044                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5045                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5046                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5047                                           item_size), 0, SOCKET_ID_ANY);
5048                 if (!sfx_actions)
5049                         return rte_flow_error_set(error, ENOMEM,
5050                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5051                                                   NULL, "no memory to split "
5052                                                   "sample flow");
5053                 /* The representor_id is -1 for uplink. */
5054                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5055                 if (!fdb_tx)
5056                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5057                                         + act_size);
5058                 pre_actions = sfx_actions + actions_n;
5059                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5060                                                 actions, sfx_actions,
5061                                                 pre_actions, actions_n,
5062                                                 sample_action_pos,
5063                                                 qrss_action_pos, error);
5064                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5065                         ret = -rte_errno;
5066                         goto exit;
5067                 }
5068                 /* Add the prefix subflow. */
5069                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5070                                               items, pre_actions,
5071                                               flow_split_info, error);
5072                 if (ret) {
5073                         ret = -rte_errno;
5074                         goto exit;
5075                 }
5076                 dev_flow->handle->split_flow_id = tag_id;
5077 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5078                 /* Set the sfx group attr. */
5079                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5080                                         dev_flow->dv.sample_res;
5081                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5082                                         sample_res->normal_path_tbl;
5083                 sfx_tbl_data = container_of(sfx_tbl,
5084                                         struct mlx5_flow_tbl_data_entry, tbl);
5085                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5086                 sfx_attr.group = sfx_attr.transfer ?
5087                                         (sfx_table_key.table_id - 1) :
5088                                          sfx_table_key.table_id;
5089                 flow_split_info->prefix_layers =
5090                                 flow_get_prefix_layer_flags(dev_flow);
5091                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5092                 /* Suffix group level already be scaled with factor, set
5093                  * skip_scale to 1 to avoid scale again in translation.
5094                  */
5095                 flow_split_info->skip_scale = 1;
5096 #endif
5097         }
5098         /* Add the suffix subflow. */
5099         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5100                                       sfx_items ? sfx_items : items,
5101                                       sfx_actions ? sfx_actions : actions,
5102                                       flow_split_info, error);
5103 exit:
5104         if (sfx_actions)
5105                 mlx5_free(sfx_actions);
5106         return ret;
5107 }
5108
5109 /**
5110  * Split the flow to subflow set. The splitters might be linked
5111  * in the chain, like this:
5112  * flow_create_split_outer() calls:
5113  *   flow_create_split_meter() calls:
5114  *     flow_create_split_metadata(meter_subflow_0) calls:
5115  *       flow_create_split_inner(metadata_subflow_0)
5116  *       flow_create_split_inner(metadata_subflow_1)
5117  *       flow_create_split_inner(metadata_subflow_2)
5118  *     flow_create_split_metadata(meter_subflow_1) calls:
5119  *       flow_create_split_inner(metadata_subflow_0)
5120  *       flow_create_split_inner(metadata_subflow_1)
5121  *       flow_create_split_inner(metadata_subflow_2)
5122  *
5123  * This provide flexible way to add new levels of flow splitting.
5124  * The all of successfully created subflows are included to the
5125  * parent flow dev_flow list.
5126  *
5127  * @param dev
5128  *   Pointer to Ethernet device.
5129  * @param[in] flow
5130  *   Parent flow structure pointer.
5131  * @param[in] attr
5132  *   Flow rule attributes.
5133  * @param[in] items
5134  *   Pattern specification (list terminated by the END pattern item).
5135  * @param[in] actions
5136  *   Associated actions (list terminated by the END action).
5137  * @param[in] flow_split_info
5138  *   Pointer to flow split info structure.
5139  * @param[out] error
5140  *   Perform verbose error reporting if not NULL.
5141  * @return
5142  *   0 on success, negative value otherwise
5143  */
5144 static int
5145 flow_create_split_outer(struct rte_eth_dev *dev,
5146                         struct rte_flow *flow,
5147                         const struct rte_flow_attr *attr,
5148                         const struct rte_flow_item items[],
5149                         const struct rte_flow_action actions[],
5150                         struct mlx5_flow_split_info *flow_split_info,
5151                         struct rte_flow_error *error)
5152 {
5153         int ret;
5154
5155         ret = flow_create_split_sample(dev, flow, attr, items,
5156                                        actions, flow_split_info, error);
5157         MLX5_ASSERT(ret <= 0);
5158         return ret;
5159 }
5160
5161 static struct mlx5_flow_tunnel *
5162 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5163                       const struct rte_flow_attr *attr,
5164                       const struct rte_flow_item items[],
5165                       const struct rte_flow_action actions[])
5166 {
5167         struct mlx5_flow_tunnel *tunnel;
5168
5169 #pragma GCC diagnostic push
5170 #pragma GCC diagnostic ignored "-Wcast-qual"
5171         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5172                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5173         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5174                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5175         else
5176                 tunnel = NULL;
5177 #pragma GCC diagnostic pop
5178
5179         return tunnel;
5180 }
5181
5182 /**
5183  * Adjust flow RSS workspace if needed.
5184  *
5185  * @param wks
5186  *   Pointer to thread flow work space.
5187  * @param rss_desc
5188  *   Pointer to RSS descriptor.
5189  * @param[in] nrssq_num
5190  *   New RSS queue number.
5191  *
5192  * @return
5193  *   0 on success, -1 otherwise and rte_errno is set.
5194  */
5195 static int
5196 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
5197                           struct mlx5_flow_rss_desc *rss_desc,
5198                           uint32_t nrssq_num)
5199 {
5200         if (likely(nrssq_num <= wks->rssq_num))
5201                 return 0;
5202         rss_desc->queue = realloc(rss_desc->queue,
5203                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
5204         if (!rss_desc->queue) {
5205                 rte_errno = ENOMEM;
5206                 return -1;
5207         }
5208         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
5209         return 0;
5210 }
5211
5212 /**
5213  * Create a flow and add it to @p list.
5214  *
5215  * @param dev
5216  *   Pointer to Ethernet device.
5217  * @param list
5218  *   Pointer to a TAILQ flow list. If this parameter NULL,
5219  *   no list insertion occurred, flow is just created,
5220  *   this is caller's responsibility to track the
5221  *   created flow.
5222  * @param[in] attr
5223  *   Flow rule attributes.
5224  * @param[in] items
5225  *   Pattern specification (list terminated by the END pattern item).
5226  * @param[in] actions
5227  *   Associated actions (list terminated by the END action).
5228  * @param[in] external
5229  *   This flow rule is created by request external to PMD.
5230  * @param[out] error
5231  *   Perform verbose error reporting if not NULL.
5232  *
5233  * @return
5234  *   A flow index on success, 0 otherwise and rte_errno is set.
5235  */
5236 static uint32_t
5237 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5238                  const struct rte_flow_attr *attr,
5239                  const struct rte_flow_item items[],
5240                  const struct rte_flow_action original_actions[],
5241                  bool external, struct rte_flow_error *error)
5242 {
5243         struct mlx5_priv *priv = dev->data->dev_private;
5244         struct rte_flow *flow = NULL;
5245         struct mlx5_flow *dev_flow;
5246         const struct rte_flow_action_rss *rss;
5247         struct mlx5_translated_shared_action
5248                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5249         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5250         union {
5251                 struct mlx5_flow_expand_rss buf;
5252                 uint8_t buffer[2048];
5253         } expand_buffer;
5254         union {
5255                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5256                 uint8_t buffer[2048];
5257         } actions_rx;
5258         union {
5259                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5260                 uint8_t buffer[2048];
5261         } actions_hairpin_tx;
5262         union {
5263                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5264                 uint8_t buffer[2048];
5265         } items_tx;
5266         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5267         struct mlx5_flow_rss_desc *rss_desc;
5268         const struct rte_flow_action *p_actions_rx;
5269         uint32_t i;
5270         uint32_t idx = 0;
5271         int hairpin_flow;
5272         struct rte_flow_attr attr_tx = { .priority = 0 };
5273         const struct rte_flow_action *actions;
5274         struct rte_flow_action *translated_actions = NULL;
5275         struct mlx5_flow_tunnel *tunnel;
5276         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5277         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
5278         struct mlx5_flow_split_info flow_split_info = {
5279                 .external = !!external,
5280                 .skip_scale = 0,
5281                 .flow_idx = 0,
5282                 .prefix_mark = 0,
5283                 .prefix_layers = 0
5284         };
5285         int ret;
5286
5287         MLX5_ASSERT(wks);
5288         rss_desc = &wks->rss_desc;
5289         ret = flow_shared_actions_translate(dev, original_actions,
5290                                             shared_actions,
5291                                             &shared_actions_n,
5292                                             &translated_actions, error);
5293         if (ret < 0) {
5294                 MLX5_ASSERT(translated_actions == NULL);
5295                 return 0;
5296         }
5297         actions = translated_actions ? translated_actions : original_actions;
5298         p_actions_rx = actions;
5299         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5300         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
5301                                 external, hairpin_flow, error);
5302         if (ret < 0)
5303                 goto error_before_hairpin_split;
5304         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5305         if (!flow) {
5306                 rte_errno = ENOMEM;
5307                 goto error_before_hairpin_split;
5308         }
5309         if (hairpin_flow > 0) {
5310                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5311                         rte_errno = EINVAL;
5312                         goto error_before_hairpin_split;
5313                 }
5314                 flow_hairpin_split(dev, actions, actions_rx.actions,
5315                                    actions_hairpin_tx.actions, items_tx.items,
5316                                    idx);
5317                 p_actions_rx = actions_rx.actions;
5318         }
5319         flow_split_info.flow_idx = idx;
5320         flow->drv_type = flow_get_drv_type(dev, attr);
5321         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5322                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5323         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
5324         rss = flow_get_rss_action(p_actions_rx);
5325         if (rss) {
5326                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
5327                         return 0;
5328                 /*
5329                  * The following information is required by
5330                  * mlx5_flow_hashfields_adjust() in advance.
5331                  */
5332                 rss_desc->level = rss->level;
5333                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5334                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5335         }
5336         flow->dev_handles = 0;
5337         if (rss && rss->types) {
5338                 unsigned int graph_root;
5339
5340                 graph_root = find_graph_root(items, rss->level);
5341                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5342                                            items, rss->types,
5343                                            mlx5_support_expansion, graph_root);
5344                 MLX5_ASSERT(ret > 0 &&
5345                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5346         } else {
5347                 buf->entries = 1;
5348                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5349         }
5350         rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions,
5351                                                       shared_actions_n);
5352         for (i = 0; i < buf->entries; ++i) {
5353                 /* Initialize flow split data. */
5354                 flow_split_info.prefix_layers = 0;
5355                 flow_split_info.prefix_mark = 0;
5356                 flow_split_info.skip_scale = 0;
5357                 /*
5358                  * The splitter may create multiple dev_flows,
5359                  * depending on configuration. In the simplest
5360                  * case it just creates unmodified original flow.
5361                  */
5362                 ret = flow_create_split_outer(dev, flow, attr,
5363                                               buf->entry[i].pattern,
5364                                               p_actions_rx, &flow_split_info,
5365                                               error);
5366                 if (ret < 0)
5367                         goto error;
5368                 if (is_flow_tunnel_steer_rule(dev, attr,
5369                                               buf->entry[i].pattern,
5370                                               p_actions_rx)) {
5371                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5372                                                            p_actions_rx,
5373                                                            idx,
5374                                                            &default_miss_ctx,
5375                                                            error);
5376                         if (ret < 0) {
5377                                 mlx5_free(default_miss_ctx.queue);
5378                                 goto error;
5379                         }
5380                 }
5381         }
5382         /* Create the tx flow. */
5383         if (hairpin_flow) {
5384                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5385                 attr_tx.ingress = 0;
5386                 attr_tx.egress = 1;
5387                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5388                                          actions_hairpin_tx.actions,
5389                                          idx, error);
5390                 if (!dev_flow)
5391                         goto error;
5392                 dev_flow->flow = flow;
5393                 dev_flow->external = 0;
5394                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5395                               dev_flow->handle, next);
5396                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5397                                          items_tx.items,
5398                                          actions_hairpin_tx.actions, error);
5399                 if (ret < 0)
5400                         goto error;
5401         }
5402         /*
5403          * Update the metadata register copy table. If extensive
5404          * metadata feature is enabled and registers are supported
5405          * we might create the extra rte_flow for each unique
5406          * MARK/FLAG action ID.
5407          *
5408          * The table is updated for ingress Flows only, because
5409          * the egress Flows belong to the different device and
5410          * copy table should be updated in peer NIC Rx domain.
5411          */
5412         if (attr->ingress &&
5413             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5414                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5415                 if (ret)
5416                         goto error;
5417         }
5418         /*
5419          * If the flow is external (from application) OR device is started,
5420          * OR mreg discover, then apply immediately.
5421          */
5422         if (external || dev->data->dev_started ||
5423             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
5424              attr->priority == MLX5_FLOW_PRIO_RSVD)) {
5425                 ret = flow_drv_apply(dev, flow, error);
5426                 if (ret < 0)
5427                         goto error;
5428         }
5429         if (list) {
5430                 rte_spinlock_lock(&priv->flow_list_lock);
5431                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5432                              flow, next);
5433                 rte_spinlock_unlock(&priv->flow_list_lock);
5434         }
5435         flow_rxq_flags_set(dev, flow);
5436         rte_free(translated_actions);
5437         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5438         if (tunnel) {
5439                 flow->tunnel = 1;
5440                 flow->tunnel_id = tunnel->tunnel_id;
5441                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5442                 mlx5_free(default_miss_ctx.queue);
5443         }
5444         mlx5_flow_pop_thread_workspace();
5445         return idx;
5446 error:
5447         MLX5_ASSERT(flow);
5448         ret = rte_errno; /* Save rte_errno before cleanup. */
5449         flow_mreg_del_copy_action(dev, flow);
5450         flow_drv_destroy(dev, flow);
5451         if (rss_desc->shared_rss)
5452                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
5453                         mlx5_ipool_get
5454                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
5455                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
5456         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5457         rte_errno = ret; /* Restore rte_errno. */
5458         ret = rte_errno;
5459         rte_errno = ret;
5460         mlx5_flow_pop_thread_workspace();
5461 error_before_hairpin_split:
5462         rte_free(translated_actions);
5463         return 0;
5464 }
5465
5466 /**
5467  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5468  * incoming packets to table 1.
5469  *
5470  * Other flow rules, requested for group n, will be created in
5471  * e-switch table n+1.
5472  * Jump action to e-switch group n will be created to group n+1.
5473  *
5474  * Used when working in switchdev mode, to utilise advantages of table 1
5475  * and above.
5476  *
5477  * @param dev
5478  *   Pointer to Ethernet device.
5479  *
5480  * @return
5481  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5482  */
5483 struct rte_flow *
5484 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5485 {
5486         const struct rte_flow_attr attr = {
5487                 .group = 0,
5488                 .priority = 0,
5489                 .ingress = 1,
5490                 .egress = 0,
5491                 .transfer = 1,
5492         };
5493         const struct rte_flow_item pattern = {
5494                 .type = RTE_FLOW_ITEM_TYPE_END,
5495         };
5496         struct rte_flow_action_jump jump = {
5497                 .group = 1,
5498         };
5499         const struct rte_flow_action actions[] = {
5500                 {
5501                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5502                         .conf = &jump,
5503                 },
5504                 {
5505                         .type = RTE_FLOW_ACTION_TYPE_END,
5506                 },
5507         };
5508         struct mlx5_priv *priv = dev->data->dev_private;
5509         struct rte_flow_error error;
5510
5511         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5512                                                    &attr, &pattern,
5513                                                    actions, false, &error);
5514 }
5515
5516 /**
5517  * Validate a flow supported by the NIC.
5518  *
5519  * @see rte_flow_validate()
5520  * @see rte_flow_ops
5521  */
5522 int
5523 mlx5_flow_validate(struct rte_eth_dev *dev,
5524                    const struct rte_flow_attr *attr,
5525                    const struct rte_flow_item items[],
5526                    const struct rte_flow_action original_actions[],
5527                    struct rte_flow_error *error)
5528 {
5529         int hairpin_flow;
5530         struct mlx5_translated_shared_action
5531                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5532         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5533         const struct rte_flow_action *actions;
5534         struct rte_flow_action *translated_actions = NULL;
5535         int ret = flow_shared_actions_translate(dev, original_actions,
5536                                                 shared_actions,
5537                                                 &shared_actions_n,
5538                                                 &translated_actions, error);
5539
5540         if (ret)
5541                 return ret;
5542         actions = translated_actions ? translated_actions : original_actions;
5543         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5544         ret = flow_drv_validate(dev, attr, items, actions,
5545                                 true, hairpin_flow, error);
5546         rte_free(translated_actions);
5547         return ret;
5548 }
5549
5550 /**
5551  * Create a flow.
5552  *
5553  * @see rte_flow_create()
5554  * @see rte_flow_ops
5555  */
5556 struct rte_flow *
5557 mlx5_flow_create(struct rte_eth_dev *dev,
5558                  const struct rte_flow_attr *attr,
5559                  const struct rte_flow_item items[],
5560                  const struct rte_flow_action actions[],
5561                  struct rte_flow_error *error)
5562 {
5563         struct mlx5_priv *priv = dev->data->dev_private;
5564
5565         /*
5566          * If the device is not started yet, it is not allowed to created a
5567          * flow from application. PMD default flows and traffic control flows
5568          * are not affected.
5569          */
5570         if (unlikely(!dev->data->dev_started)) {
5571                 DRV_LOG(DEBUG, "port %u is not started when "
5572                         "inserting a flow", dev->data->port_id);
5573                 rte_flow_error_set(error, ENODEV,
5574                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5575                                    NULL,
5576                                    "port not started");
5577                 return NULL;
5578         }
5579
5580         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5581                                   attr, items, actions, true, error);
5582 }
5583
5584 /**
5585  * Destroy a flow in a list.
5586  *
5587  * @param dev
5588  *   Pointer to Ethernet device.
5589  * @param list
5590  *   Pointer to the Indexed flow list. If this parameter NULL,
5591  *   there is no flow removal from the list. Be noted that as
5592  *   flow is add to the indexed list, memory of the indexed
5593  *   list points to maybe changed as flow destroyed.
5594  * @param[in] flow_idx
5595  *   Index of flow to destroy.
5596  */
5597 static void
5598 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5599                   uint32_t flow_idx)
5600 {
5601         struct mlx5_priv *priv = dev->data->dev_private;
5602         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5603                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5604
5605         if (!flow)
5606                 return;
5607         /*
5608          * Update RX queue flags only if port is started, otherwise it is
5609          * already clean.
5610          */
5611         if (dev->data->dev_started)
5612                 flow_rxq_flags_trim(dev, flow);
5613         flow_drv_destroy(dev, flow);
5614         if (list) {
5615                 rte_spinlock_lock(&priv->flow_list_lock);
5616                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5617                              flow_idx, flow, next);
5618                 rte_spinlock_unlock(&priv->flow_list_lock);
5619         }
5620         if (flow->tunnel) {
5621                 struct mlx5_flow_tunnel *tunnel;
5622
5623                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5624                 RTE_VERIFY(tunnel);
5625                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5626                         mlx5_flow_tunnel_free(dev, tunnel);
5627         }
5628         flow_mreg_del_copy_action(dev, flow);
5629         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5630 }
5631
5632 /**
5633  * Destroy all flows.
5634  *
5635  * @param dev
5636  *   Pointer to Ethernet device.
5637  * @param list
5638  *   Pointer to the Indexed flow list.
5639  * @param active
5640  *   If flushing is called avtively.
5641  */
5642 void
5643 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5644 {
5645         uint32_t num_flushed = 0;
5646
5647         while (*list) {
5648                 flow_list_destroy(dev, list, *list);
5649                 num_flushed++;
5650         }
5651         if (active) {
5652                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5653                         dev->data->port_id, num_flushed);
5654         }
5655 }
5656
5657 /**
5658  * Stop all default actions for flows.
5659  *
5660  * @param dev
5661  *   Pointer to Ethernet device.
5662  */
5663 void
5664 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5665 {
5666         flow_mreg_del_default_copy_action(dev);
5667         flow_rxq_flags_clear(dev);
5668 }
5669
5670 /**
5671  * Start all default actions for flows.
5672  *
5673  * @param dev
5674  *   Pointer to Ethernet device.
5675  * @return
5676  *   0 on success, a negative errno value otherwise and rte_errno is set.
5677  */
5678 int
5679 mlx5_flow_start_default(struct rte_eth_dev *dev)
5680 {
5681         struct rte_flow_error error;
5682
5683         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5684         return flow_mreg_add_default_copy_action(dev, &error);
5685 }
5686
5687 /**
5688  * Release key of thread specific flow workspace data.
5689  */
5690 static void
5691 flow_release_workspace(void *data)
5692 {
5693         struct mlx5_flow_workspace *wks = data;
5694         struct mlx5_flow_workspace *next;
5695
5696         while (wks) {
5697                 next = wks->next;
5698                 free(wks->rss_desc.queue);
5699                 free(wks);
5700                 wks = next;
5701         }
5702 }
5703
5704 /**
5705  * Initialize key of thread specific flow workspace data.
5706  */
5707 static void
5708 flow_alloc_workspace(void)
5709 {
5710         if (pthread_key_create(&key_workspace, flow_release_workspace))
5711                 DRV_LOG(ERR, "Can't create flow workspace data thread key.");
5712 }
5713
5714 /**
5715  * Get thread specific current flow workspace.
5716  *
5717  * @return pointer to thread specific flow workspace data, NULL on error.
5718  */
5719 struct mlx5_flow_workspace*
5720 mlx5_flow_get_thread_workspace(void)
5721 {
5722         struct mlx5_flow_workspace *data;
5723
5724         data = pthread_getspecific(key_workspace);
5725         MLX5_ASSERT(data && data->inuse);
5726         if (!data || !data->inuse)
5727                 DRV_LOG(ERR, "flow workspace not initialized.");
5728         return data;
5729 }
5730
5731 /**
5732  * Allocate and init new flow workspace.
5733  *
5734  * @return pointer to flow workspace data, NULL on error.
5735  */
5736 static struct mlx5_flow_workspace*
5737 flow_alloc_thread_workspace(void)
5738 {
5739         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
5740
5741         if (!data) {
5742                 DRV_LOG(ERR, "Failed to allocate flow workspace "
5743                         "memory.");
5744                 return NULL;
5745         }
5746         data->rss_desc.queue = calloc(1,
5747                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
5748         if (!data->rss_desc.queue)
5749                 goto err;
5750         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
5751         return data;
5752 err:
5753         if (data->rss_desc.queue)
5754                 free(data->rss_desc.queue);
5755         free(data);
5756         return NULL;
5757 }
5758
5759 /**
5760  * Get new thread specific flow workspace.
5761  *
5762  * If current workspace inuse, create new one and set as current.
5763  *
5764  * @return pointer to thread specific flow workspace data, NULL on error.
5765  */
5766 static struct mlx5_flow_workspace*
5767 mlx5_flow_push_thread_workspace(void)
5768 {
5769         struct mlx5_flow_workspace *curr;
5770         struct mlx5_flow_workspace *data;
5771
5772         if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
5773                 DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
5774                 return NULL;
5775         }
5776         curr = pthread_getspecific(key_workspace);
5777         if (!curr) {
5778                 data = flow_alloc_thread_workspace();
5779                 if (!data)
5780                         return NULL;
5781         } else if (!curr->inuse) {
5782                 data = curr;
5783         } else if (curr->next) {
5784                 data = curr->next;
5785         } else {
5786                 data = flow_alloc_thread_workspace();
5787                 if (!data)
5788                         return NULL;
5789                 curr->next = data;
5790                 data->prev = curr;
5791         }
5792         data->inuse = 1;
5793         data->flow_idx = 0;
5794         /* Set as current workspace */
5795         if (pthread_setspecific(key_workspace, data))
5796                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5797         return data;
5798 }
5799
5800 /**
5801  * Close current thread specific flow workspace.
5802  *
5803  * If previous workspace available, set it as current.
5804  *
5805  * @return pointer to thread specific flow workspace data, NULL on error.
5806  */
5807 static void
5808 mlx5_flow_pop_thread_workspace(void)
5809 {
5810         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
5811
5812         if (!data)
5813                 return;
5814         if (!data->inuse) {
5815                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
5816                 return;
5817         }
5818         data->inuse = 0;
5819         if (!data->prev)
5820                 return;
5821         if (pthread_setspecific(key_workspace, data->prev))
5822                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
5823 }
5824
5825 /**
5826  * Verify the flow list is empty
5827  *
5828  * @param dev
5829  *  Pointer to Ethernet device.
5830  *
5831  * @return the number of flows not released.
5832  */
5833 int
5834 mlx5_flow_verify(struct rte_eth_dev *dev)
5835 {
5836         struct mlx5_priv *priv = dev->data->dev_private;
5837         struct rte_flow *flow;
5838         uint32_t idx;
5839         int ret = 0;
5840
5841         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5842                       flow, next) {
5843                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5844                         dev->data->port_id, (void *)flow);
5845                 ++ret;
5846         }
5847         return ret;
5848 }
5849
5850 /**
5851  * Enable default hairpin egress flow.
5852  *
5853  * @param dev
5854  *   Pointer to Ethernet device.
5855  * @param queue
5856  *   The queue index.
5857  *
5858  * @return
5859  *   0 on success, a negative errno value otherwise and rte_errno is set.
5860  */
5861 int
5862 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5863                             uint32_t queue)
5864 {
5865         struct mlx5_priv *priv = dev->data->dev_private;
5866         const struct rte_flow_attr attr = {
5867                 .egress = 1,
5868                 .priority = 0,
5869         };
5870         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5871                 .queue = queue,
5872         };
5873         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5874                 .queue = UINT32_MAX,
5875         };
5876         struct rte_flow_item items[] = {
5877                 {
5878                         .type = (enum rte_flow_item_type)
5879                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5880                         .spec = &queue_spec,
5881                         .last = NULL,
5882                         .mask = &queue_mask,
5883                 },
5884                 {
5885                         .type = RTE_FLOW_ITEM_TYPE_END,
5886                 },
5887         };
5888         struct rte_flow_action_jump jump = {
5889                 .group = MLX5_HAIRPIN_TX_TABLE,
5890         };
5891         struct rte_flow_action actions[2];
5892         uint32_t flow_idx;
5893         struct rte_flow_error error;
5894
5895         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5896         actions[0].conf = &jump;
5897         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5898         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5899                                 &attr, items, actions, false, &error);
5900         if (!flow_idx) {
5901                 DRV_LOG(DEBUG,
5902                         "Failed to create ctrl flow: rte_errno(%d),"
5903                         " type(%d), message(%s)",
5904                         rte_errno, error.type,
5905                         error.message ? error.message : " (no stated reason)");
5906                 return -rte_errno;
5907         }
5908         return 0;
5909 }
5910
5911 /**
5912  * Enable a control flow configured from the control plane.
5913  *
5914  * @param dev
5915  *   Pointer to Ethernet device.
5916  * @param eth_spec
5917  *   An Ethernet flow spec to apply.
5918  * @param eth_mask
5919  *   An Ethernet flow mask to apply.
5920  * @param vlan_spec
5921  *   A VLAN flow spec to apply.
5922  * @param vlan_mask
5923  *   A VLAN flow mask to apply.
5924  *
5925  * @return
5926  *   0 on success, a negative errno value otherwise and rte_errno is set.
5927  */
5928 int
5929 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5930                     struct rte_flow_item_eth *eth_spec,
5931                     struct rte_flow_item_eth *eth_mask,
5932                     struct rte_flow_item_vlan *vlan_spec,
5933                     struct rte_flow_item_vlan *vlan_mask)
5934 {
5935         struct mlx5_priv *priv = dev->data->dev_private;
5936         const struct rte_flow_attr attr = {
5937                 .ingress = 1,
5938                 .priority = MLX5_FLOW_PRIO_RSVD,
5939         };
5940         struct rte_flow_item items[] = {
5941                 {
5942                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5943                         .spec = eth_spec,
5944                         .last = NULL,
5945                         .mask = eth_mask,
5946                 },
5947                 {
5948                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5949                                               RTE_FLOW_ITEM_TYPE_END,
5950                         .spec = vlan_spec,
5951                         .last = NULL,
5952                         .mask = vlan_mask,
5953                 },
5954                 {
5955                         .type = RTE_FLOW_ITEM_TYPE_END,
5956                 },
5957         };
5958         uint16_t queue[priv->reta_idx_n];
5959         struct rte_flow_action_rss action_rss = {
5960                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5961                 .level = 0,
5962                 .types = priv->rss_conf.rss_hf,
5963                 .key_len = priv->rss_conf.rss_key_len,
5964                 .queue_num = priv->reta_idx_n,
5965                 .key = priv->rss_conf.rss_key,
5966                 .queue = queue,
5967         };
5968         struct rte_flow_action actions[] = {
5969                 {
5970                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5971                         .conf = &action_rss,
5972                 },
5973                 {
5974                         .type = RTE_FLOW_ACTION_TYPE_END,
5975                 },
5976         };
5977         uint32_t flow_idx;
5978         struct rte_flow_error error;
5979         unsigned int i;
5980
5981         if (!priv->reta_idx_n || !priv->rxqs_n) {
5982                 return 0;
5983         }
5984         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5985                 action_rss.types = 0;
5986         for (i = 0; i != priv->reta_idx_n; ++i)
5987                 queue[i] = (*priv->reta_idx)[i];
5988         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5989                                 &attr, items, actions, false, &error);
5990         if (!flow_idx)
5991                 return -rte_errno;
5992         return 0;
5993 }
5994
5995 /**
5996  * Enable a flow control configured from the control plane.
5997  *
5998  * @param dev
5999  *   Pointer to Ethernet device.
6000  * @param eth_spec
6001  *   An Ethernet flow spec to apply.
6002  * @param eth_mask
6003  *   An Ethernet flow mask to apply.
6004  *
6005  * @return
6006  *   0 on success, a negative errno value otherwise and rte_errno is set.
6007  */
6008 int
6009 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6010                struct rte_flow_item_eth *eth_spec,
6011                struct rte_flow_item_eth *eth_mask)
6012 {
6013         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6014 }
6015
6016 /**
6017  * Create default miss flow rule matching lacp traffic
6018  *
6019  * @param dev
6020  *   Pointer to Ethernet device.
6021  * @param eth_spec
6022  *   An Ethernet flow spec to apply.
6023  *
6024  * @return
6025  *   0 on success, a negative errno value otherwise and rte_errno is set.
6026  */
6027 int
6028 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6029 {
6030         struct mlx5_priv *priv = dev->data->dev_private;
6031         /*
6032          * The LACP matching is done by only using ether type since using
6033          * a multicast dst mac causes kernel to give low priority to this flow.
6034          */
6035         static const struct rte_flow_item_eth lacp_spec = {
6036                 .type = RTE_BE16(0x8809),
6037         };
6038         static const struct rte_flow_item_eth lacp_mask = {
6039                 .type = 0xffff,
6040         };
6041         const struct rte_flow_attr attr = {
6042                 .ingress = 1,
6043         };
6044         struct rte_flow_item items[] = {
6045                 {
6046                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6047                         .spec = &lacp_spec,
6048                         .mask = &lacp_mask,
6049                 },
6050                 {
6051                         .type = RTE_FLOW_ITEM_TYPE_END,
6052                 },
6053         };
6054         struct rte_flow_action actions[] = {
6055                 {
6056                         .type = (enum rte_flow_action_type)
6057                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6058                 },
6059                 {
6060                         .type = RTE_FLOW_ACTION_TYPE_END,
6061                 },
6062         };
6063         struct rte_flow_error error;
6064         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6065                                 &attr, items, actions, false, &error);
6066
6067         if (!flow_idx)
6068                 return -rte_errno;
6069         return 0;
6070 }
6071
6072 /**
6073  * Destroy a flow.
6074  *
6075  * @see rte_flow_destroy()
6076  * @see rte_flow_ops
6077  */
6078 int
6079 mlx5_flow_destroy(struct rte_eth_dev *dev,
6080                   struct rte_flow *flow,
6081                   struct rte_flow_error *error __rte_unused)
6082 {
6083         struct mlx5_priv *priv = dev->data->dev_private;
6084
6085         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6086         return 0;
6087 }
6088
6089 /**
6090  * Destroy all flows.
6091  *
6092  * @see rte_flow_flush()
6093  * @see rte_flow_ops
6094  */
6095 int
6096 mlx5_flow_flush(struct rte_eth_dev *dev,
6097                 struct rte_flow_error *error __rte_unused)
6098 {
6099         struct mlx5_priv *priv = dev->data->dev_private;
6100
6101         mlx5_flow_list_flush(dev, &priv->flows, false);
6102         return 0;
6103 }
6104
6105 /**
6106  * Isolated mode.
6107  *
6108  * @see rte_flow_isolate()
6109  * @see rte_flow_ops
6110  */
6111 int
6112 mlx5_flow_isolate(struct rte_eth_dev *dev,
6113                   int enable,
6114                   struct rte_flow_error *error)
6115 {
6116         struct mlx5_priv *priv = dev->data->dev_private;
6117
6118         if (dev->data->dev_started) {
6119                 rte_flow_error_set(error, EBUSY,
6120                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6121                                    NULL,
6122                                    "port must be stopped first");
6123                 return -rte_errno;
6124         }
6125         priv->isolated = !!enable;
6126         if (enable)
6127                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6128         else
6129                 dev->dev_ops = &mlx5_os_dev_ops;
6130
6131         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6132         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6133
6134         return 0;
6135 }
6136
6137 /**
6138  * Query a flow.
6139  *
6140  * @see rte_flow_query()
6141  * @see rte_flow_ops
6142  */
6143 static int
6144 flow_drv_query(struct rte_eth_dev *dev,
6145                uint32_t flow_idx,
6146                const struct rte_flow_action *actions,
6147                void *data,
6148                struct rte_flow_error *error)
6149 {
6150         struct mlx5_priv *priv = dev->data->dev_private;
6151         const struct mlx5_flow_driver_ops *fops;
6152         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6153                                                [MLX5_IPOOL_RTE_FLOW],
6154                                                flow_idx);
6155         enum mlx5_flow_drv_type ftype;
6156
6157         if (!flow) {
6158                 return rte_flow_error_set(error, ENOENT,
6159                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6160                           NULL,
6161                           "invalid flow handle");
6162         }
6163         ftype = flow->drv_type;
6164         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6165         fops = flow_get_drv_ops(ftype);
6166
6167         return fops->query(dev, flow, actions, data, error);
6168 }
6169
6170 /**
6171  * Query a flow.
6172  *
6173  * @see rte_flow_query()
6174  * @see rte_flow_ops
6175  */
6176 int
6177 mlx5_flow_query(struct rte_eth_dev *dev,
6178                 struct rte_flow *flow,
6179                 const struct rte_flow_action *actions,
6180                 void *data,
6181                 struct rte_flow_error *error)
6182 {
6183         int ret;
6184
6185         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6186                              error);
6187         if (ret < 0)
6188                 return ret;
6189         return 0;
6190 }
6191
6192 /**
6193  * Manage filter operations.
6194  *
6195  * @param dev
6196  *   Pointer to Ethernet device structure.
6197  * @param filter_type
6198  *   Filter type.
6199  * @param filter_op
6200  *   Operation to perform.
6201  * @param arg
6202  *   Pointer to operation-specific structure.
6203  *
6204  * @return
6205  *   0 on success, a negative errno value otherwise and rte_errno is set.
6206  */
6207 int
6208 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6209                      enum rte_filter_type filter_type,
6210                      enum rte_filter_op filter_op,
6211                      void *arg)
6212 {
6213         switch (filter_type) {
6214         case RTE_ETH_FILTER_GENERIC:
6215                 if (filter_op != RTE_ETH_FILTER_GET) {
6216                         rte_errno = EINVAL;
6217                         return -rte_errno;
6218                 }
6219                 *(const void **)arg = &mlx5_flow_ops;
6220                 return 0;
6221         default:
6222                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6223                         dev->data->port_id, filter_type);
6224                 rte_errno = ENOTSUP;
6225                 return -rte_errno;
6226         }
6227         return 0;
6228 }
6229
6230 /**
6231  * Create the needed meter and suffix tables.
6232  *
6233  * @param[in] dev
6234  *   Pointer to Ethernet device.
6235  * @param[in] fm
6236  *   Pointer to the flow meter.
6237  *
6238  * @return
6239  *   Pointer to table set on success, NULL otherwise.
6240  */
6241 struct mlx5_meter_domains_infos *
6242 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6243                           const struct mlx5_flow_meter *fm)
6244 {
6245         const struct mlx5_flow_driver_ops *fops;
6246
6247         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6248         return fops->create_mtr_tbls(dev, fm);
6249 }
6250
6251 /**
6252  * Destroy the meter table set.
6253  *
6254  * @param[in] dev
6255  *   Pointer to Ethernet device.
6256  * @param[in] tbl
6257  *   Pointer to the meter table set.
6258  *
6259  * @return
6260  *   0 on success.
6261  */
6262 int
6263 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6264                            struct mlx5_meter_domains_infos *tbls)
6265 {
6266         const struct mlx5_flow_driver_ops *fops;
6267
6268         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6269         return fops->destroy_mtr_tbls(dev, tbls);
6270 }
6271
6272 /**
6273  * Create policer rules.
6274  *
6275  * @param[in] dev
6276  *   Pointer to Ethernet device.
6277  * @param[in] fm
6278  *   Pointer to flow meter structure.
6279  * @param[in] attr
6280  *   Pointer to flow attributes.
6281  *
6282  * @return
6283  *   0 on success, -1 otherwise.
6284  */
6285 int
6286 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6287                                struct mlx5_flow_meter *fm,
6288                                const struct rte_flow_attr *attr)
6289 {
6290         const struct mlx5_flow_driver_ops *fops;
6291
6292         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6293         return fops->create_policer_rules(dev, fm, attr);
6294 }
6295
6296 /**
6297  * Destroy policer rules.
6298  *
6299  * @param[in] fm
6300  *   Pointer to flow meter structure.
6301  * @param[in] attr
6302  *   Pointer to flow attributes.
6303  *
6304  * @return
6305  *   0 on success, -1 otherwise.
6306  */
6307 int
6308 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6309                                 struct mlx5_flow_meter *fm,
6310                                 const struct rte_flow_attr *attr)
6311 {
6312         const struct mlx5_flow_driver_ops *fops;
6313
6314         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6315         return fops->destroy_policer_rules(dev, fm, attr);
6316 }
6317
6318 /**
6319  * Allocate a counter.
6320  *
6321  * @param[in] dev
6322  *   Pointer to Ethernet device structure.
6323  *
6324  * @return
6325  *   Index to allocated counter  on success, 0 otherwise.
6326  */
6327 uint32_t
6328 mlx5_counter_alloc(struct rte_eth_dev *dev)
6329 {
6330         const struct mlx5_flow_driver_ops *fops;
6331         struct rte_flow_attr attr = { .transfer = 0 };
6332
6333         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6334                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6335                 return fops->counter_alloc(dev);
6336         }
6337         DRV_LOG(ERR,
6338                 "port %u counter allocate is not supported.",
6339                  dev->data->port_id);
6340         return 0;
6341 }
6342
6343 /**
6344  * Free a counter.
6345  *
6346  * @param[in] dev
6347  *   Pointer to Ethernet device structure.
6348  * @param[in] cnt
6349  *   Index to counter to be free.
6350  */
6351 void
6352 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6353 {
6354         const struct mlx5_flow_driver_ops *fops;
6355         struct rte_flow_attr attr = { .transfer = 0 };
6356
6357         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6358                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6359                 fops->counter_free(dev, cnt);
6360                 return;
6361         }
6362         DRV_LOG(ERR,
6363                 "port %u counter free is not supported.",
6364                  dev->data->port_id);
6365 }
6366
6367 /**
6368  * Query counter statistics.
6369  *
6370  * @param[in] dev
6371  *   Pointer to Ethernet device structure.
6372  * @param[in] cnt
6373  *   Index to counter to query.
6374  * @param[in] clear
6375  *   Set to clear counter statistics.
6376  * @param[out] pkts
6377  *   The counter hits packets number to save.
6378  * @param[out] bytes
6379  *   The counter hits bytes number to save.
6380  *
6381  * @return
6382  *   0 on success, a negative errno value otherwise.
6383  */
6384 int
6385 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6386                    bool clear, uint64_t *pkts, uint64_t *bytes)
6387 {
6388         const struct mlx5_flow_driver_ops *fops;
6389         struct rte_flow_attr attr = { .transfer = 0 };
6390
6391         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6392                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6393                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6394         }
6395         DRV_LOG(ERR,
6396                 "port %u counter query is not supported.",
6397                  dev->data->port_id);
6398         return -ENOTSUP;
6399 }
6400
6401 /**
6402  * Allocate a new memory for the counter values wrapped by all the needed
6403  * management.
6404  *
6405  * @param[in] sh
6406  *   Pointer to mlx5_dev_ctx_shared object.
6407  *
6408  * @return
6409  *   0 on success, a negative errno value otherwise.
6410  */
6411 static int
6412 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
6413 {
6414         struct mlx5_devx_mkey_attr mkey_attr;
6415         struct mlx5_counter_stats_mem_mng *mem_mng;
6416         volatile struct flow_counter_stats *raw_data;
6417         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
6418         int size = (sizeof(struct flow_counter_stats) *
6419                         MLX5_COUNTERS_PER_POOL +
6420                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
6421                         sizeof(struct mlx5_counter_stats_mem_mng);
6422         size_t pgsize = rte_mem_page_size();
6423         uint8_t *mem;
6424         int i;
6425
6426         if (pgsize == (size_t)-1) {
6427                 DRV_LOG(ERR, "Failed to get mem page size");
6428                 rte_errno = ENOMEM;
6429                 return -ENOMEM;
6430         }
6431         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
6432         if (!mem) {
6433                 rte_errno = ENOMEM;
6434                 return -ENOMEM;
6435         }
6436         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
6437         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
6438         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
6439                                                  IBV_ACCESS_LOCAL_WRITE);
6440         if (!mem_mng->umem) {
6441                 rte_errno = errno;
6442                 mlx5_free(mem);
6443                 return -rte_errno;
6444         }
6445         mkey_attr.addr = (uintptr_t)mem;
6446         mkey_attr.size = size;
6447         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
6448         mkey_attr.pd = sh->pdn;
6449         mkey_attr.log_entity_size = 0;
6450         mkey_attr.pg_access = 0;
6451         mkey_attr.klm_array = NULL;
6452         mkey_attr.klm_num = 0;
6453         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
6454         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
6455         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
6456         if (!mem_mng->dm) {
6457                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
6458                 rte_errno = errno;
6459                 mlx5_free(mem);
6460                 return -rte_errno;
6461         }
6462         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
6463         raw_data = (volatile struct flow_counter_stats *)mem;
6464         for (i = 0; i < raws_n; ++i) {
6465                 mem_mng->raws[i].mem_mng = mem_mng;
6466                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
6467         }
6468         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
6469                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
6470                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
6471                                  next);
6472         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
6473         sh->cmng.mem_mng = mem_mng;
6474         return 0;
6475 }
6476
6477 /**
6478  * Set the statistic memory to the new counter pool.
6479  *
6480  * @param[in] sh
6481  *   Pointer to mlx5_dev_ctx_shared object.
6482  * @param[in] pool
6483  *   Pointer to the pool to set the statistic memory.
6484  *
6485  * @return
6486  *   0 on success, a negative errno value otherwise.
6487  */
6488 static int
6489 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
6490                                struct mlx5_flow_counter_pool *pool)
6491 {
6492         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6493         /* Resize statistic memory once used out. */
6494         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
6495             mlx5_flow_create_counter_stat_mem_mng(sh)) {
6496                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
6497                 return -1;
6498         }
6499         rte_spinlock_lock(&pool->sl);
6500         pool->raw = cmng->mem_mng->raws + pool->index %
6501                     MLX5_CNT_CONTAINER_RESIZE;
6502         rte_spinlock_unlock(&pool->sl);
6503         pool->raw_hw = NULL;
6504         return 0;
6505 }
6506
6507 #define MLX5_POOL_QUERY_FREQ_US 1000000
6508
6509 /**
6510  * Set the periodic procedure for triggering asynchronous batch queries for all
6511  * the counter pools.
6512  *
6513  * @param[in] sh
6514  *   Pointer to mlx5_dev_ctx_shared object.
6515  */
6516 void
6517 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6518 {
6519         uint32_t pools_n, us;
6520
6521         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
6522         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6523         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6524         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6525                 sh->cmng.query_thread_on = 0;
6526                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6527         } else {
6528                 sh->cmng.query_thread_on = 1;
6529         }
6530 }
6531
6532 /**
6533  * The periodic procedure for triggering asynchronous batch queries for all the
6534  * counter pools. This function is probably called by the host thread.
6535  *
6536  * @param[in] arg
6537  *   The parameter for the alarm process.
6538  */
6539 void
6540 mlx5_flow_query_alarm(void *arg)
6541 {
6542         struct mlx5_dev_ctx_shared *sh = arg;
6543         int ret;
6544         uint16_t pool_index = sh->cmng.pool_index;
6545         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6546         struct mlx5_flow_counter_pool *pool;
6547         uint16_t n_valid;
6548
6549         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6550                 goto set_alarm;
6551         rte_spinlock_lock(&cmng->pool_update_sl);
6552         pool = cmng->pools[pool_index];
6553         n_valid = cmng->n_valid;
6554         rte_spinlock_unlock(&cmng->pool_update_sl);
6555         /* Set the statistic memory to the new created pool. */
6556         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
6557                 goto set_alarm;
6558         if (pool->raw_hw)
6559                 /* There is a pool query in progress. */
6560                 goto set_alarm;
6561         pool->raw_hw =
6562                 LIST_FIRST(&sh->cmng.free_stat_raws);
6563         if (!pool->raw_hw)
6564                 /* No free counter statistics raw memory. */
6565                 goto set_alarm;
6566         /*
6567          * Identify the counters released between query trigger and query
6568          * handle more efficiently. The counter released in this gap period
6569          * should wait for a new round of query as the new arrived packets
6570          * will not be taken into account.
6571          */
6572         pool->query_gen++;
6573         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
6574                                                MLX5_COUNTERS_PER_POOL,
6575                                                NULL, NULL,
6576                                                pool->raw_hw->mem_mng->dm->id,
6577                                                (void *)(uintptr_t)
6578                                                pool->raw_hw->data,
6579                                                sh->devx_comp,
6580                                                (uint64_t)(uintptr_t)pool);
6581         if (ret) {
6582                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6583                         " %d", pool->min_dcs->id);
6584                 pool->raw_hw = NULL;
6585                 goto set_alarm;
6586         }
6587         LIST_REMOVE(pool->raw_hw, next);
6588         sh->cmng.pending_queries++;
6589         pool_index++;
6590         if (pool_index >= n_valid)
6591                 pool_index = 0;
6592 set_alarm:
6593         sh->cmng.pool_index = pool_index;
6594         mlx5_set_query_alarm(sh);
6595 }
6596
6597 /**
6598  * Check and callback event for new aged flow in the counter pool
6599  *
6600  * @param[in] sh
6601  *   Pointer to mlx5_dev_ctx_shared object.
6602  * @param[in] pool
6603  *   Pointer to Current counter pool.
6604  */
6605 static void
6606 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6607                    struct mlx5_flow_counter_pool *pool)
6608 {
6609         struct mlx5_priv *priv;
6610         struct mlx5_flow_counter *cnt;
6611         struct mlx5_age_info *age_info;
6612         struct mlx5_age_param *age_param;
6613         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6614         struct mlx5_counter_stats_raw *prev = pool->raw;
6615         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
6616         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
6617         uint16_t expected = AGE_CANDIDATE;
6618         uint32_t i;
6619
6620         pool->time_of_last_age_check = curr_time;
6621         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6622                 cnt = MLX5_POOL_GET_CNT(pool, i);
6623                 age_param = MLX5_CNT_TO_AGE(cnt);
6624                 if (__atomic_load_n(&age_param->state,
6625                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
6626                         continue;
6627                 if (cur->data[i].hits != prev->data[i].hits) {
6628                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
6629                                          __ATOMIC_RELAXED);
6630                         continue;
6631                 }
6632                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
6633                                        time_delta,
6634                                        __ATOMIC_RELAXED) <= age_param->timeout)
6635                         continue;
6636                 /**
6637                  * Hold the lock first, or if between the
6638                  * state AGE_TMOUT and tailq operation the
6639                  * release happened, the release procedure
6640                  * may delete a non-existent tailq node.
6641                  */
6642                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6643                 age_info = GET_PORT_AGE_INFO(priv);
6644                 rte_spinlock_lock(&age_info->aged_sl);
6645                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
6646                                                 AGE_TMOUT, false,
6647                                                 __ATOMIC_RELAXED,
6648                                                 __ATOMIC_RELAXED)) {
6649                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6650                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6651                 }
6652                 rte_spinlock_unlock(&age_info->aged_sl);
6653         }
6654         mlx5_age_event_prepare(sh);
6655 }
6656
6657 /**
6658  * Handler for the HW respond about ready values from an asynchronous batch
6659  * query. This function is probably called by the host thread.
6660  *
6661  * @param[in] sh
6662  *   The pointer to the shared device context.
6663  * @param[in] async_id
6664  *   The Devx async ID.
6665  * @param[in] status
6666  *   The status of the completion.
6667  */
6668 void
6669 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6670                                   uint64_t async_id, int status)
6671 {
6672         struct mlx5_flow_counter_pool *pool =
6673                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6674         struct mlx5_counter_stats_raw *raw_to_free;
6675         uint8_t query_gen = pool->query_gen ^ 1;
6676         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
6677         enum mlx5_counter_type cnt_type =
6678                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6679                                 MLX5_COUNTER_TYPE_ORIGIN;
6680
6681         if (unlikely(status)) {
6682                 raw_to_free = pool->raw_hw;
6683         } else {
6684                 raw_to_free = pool->raw;
6685                 if (pool->is_aged)
6686                         mlx5_flow_aging_check(sh, pool);
6687                 rte_spinlock_lock(&pool->sl);
6688                 pool->raw = pool->raw_hw;
6689                 rte_spinlock_unlock(&pool->sl);
6690                 /* Be sure the new raw counters data is updated in memory. */
6691                 rte_io_wmb();
6692                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6693                         rte_spinlock_lock(&cmng->csl[cnt_type]);
6694                         TAILQ_CONCAT(&cmng->counters[cnt_type],
6695                                      &pool->counters[query_gen], next);
6696                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6697                 }
6698         }
6699         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6700         pool->raw_hw = NULL;
6701         sh->cmng.pending_queries--;
6702 }
6703
6704 static int
6705 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
6706                     const struct flow_grp_info *grp_info,
6707                     struct rte_flow_error *error)
6708 {
6709         if (grp_info->transfer && grp_info->external &&
6710             grp_info->fdb_def_rule) {
6711                 if (group == UINT32_MAX)
6712                         return rte_flow_error_set
6713                                                 (error, EINVAL,
6714                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6715                                                  NULL,
6716                                                  "group index not supported");
6717                 *table = group + 1;
6718         } else {
6719                 *table = group;
6720         }
6721         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
6722         return 0;
6723 }
6724
6725 /**
6726  * Translate the rte_flow group index to HW table value.
6727  *
6728  * If tunnel offload is disabled, all group ids converted to flow table
6729  * id using the standard method.
6730  * If tunnel offload is enabled, group id can be converted using the
6731  * standard or tunnel conversion method. Group conversion method
6732  * selection depends on flags in `grp_info` parameter:
6733  * - Internal (grp_info.external == 0) groups conversion uses the
6734  *   standard method.
6735  * - Group ids in JUMP action converted with the tunnel conversion.
6736  * - Group id in rule attribute conversion depends on a rule type and
6737  *   group id value:
6738  *   ** non zero group attributes converted with the tunnel method
6739  *   ** zero group attribute in non-tunnel rule is converted using the
6740  *      standard method - there's only one root table
6741  *   ** zero group attribute in steer tunnel rule is converted with the
6742  *      standard method - single root table
6743  *   ** zero group attribute in match tunnel rule is a special OvS
6744  *      case: that value is used for portability reasons. That group
6745  *      id is converted with the tunnel conversion method.
6746  *
6747  * @param[in] dev
6748  *   Port device
6749  * @param[in] tunnel
6750  *   PMD tunnel offload object
6751  * @param[in] group
6752  *   rte_flow group index value.
6753  * @param[out] table
6754  *   HW table value.
6755  * @param[in] grp_info
6756  *   flags used for conversion
6757  * @param[out] error
6758  *   Pointer to error structure.
6759  *
6760  * @return
6761  *   0 on success, a negative errno value otherwise and rte_errno is set.
6762  */
6763 int
6764 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
6765                          const struct mlx5_flow_tunnel *tunnel,
6766                          uint32_t group, uint32_t *table,
6767                          const struct flow_grp_info *grp_info,
6768                          struct rte_flow_error *error)
6769 {
6770         int ret;
6771         bool standard_translation;
6772
6773         if (!grp_info->skip_scale && grp_info->external &&
6774             group < MLX5_MAX_TABLES_EXTERNAL)
6775                 group *= MLX5_FLOW_TABLE_FACTOR;
6776         if (is_tunnel_offload_active(dev)) {
6777                 standard_translation = !grp_info->external ||
6778                                         grp_info->std_tbl_fix;
6779         } else {
6780                 standard_translation = true;
6781         }
6782         DRV_LOG(DEBUG,
6783                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
6784                 dev->data->port_id, group, grp_info->transfer,
6785                 grp_info->external, grp_info->fdb_def_rule,
6786                 standard_translation ? "STANDARD" : "TUNNEL");
6787         if (standard_translation)
6788                 ret = flow_group_to_table(dev->data->port_id, group, table,
6789                                           grp_info, error);
6790         else
6791                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
6792                                                       table, error);
6793
6794         return ret;
6795 }
6796
6797 /**
6798  * Discover availability of metadata reg_c's.
6799  *
6800  * Iteratively use test flows to check availability.
6801  *
6802  * @param[in] dev
6803  *   Pointer to the Ethernet device structure.
6804  *
6805  * @return
6806  *   0 on success, a negative errno value otherwise and rte_errno is set.
6807  */
6808 int
6809 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6810 {
6811         struct mlx5_priv *priv = dev->data->dev_private;
6812         struct mlx5_dev_config *config = &priv->config;
6813         enum modify_reg idx;
6814         int n = 0;
6815
6816         /* reg_c[0] and reg_c[1] are reserved. */
6817         config->flow_mreg_c[n++] = REG_C_0;
6818         config->flow_mreg_c[n++] = REG_C_1;
6819         /* Discover availability of other reg_c's. */
6820         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6821                 struct rte_flow_attr attr = {
6822                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6823                         .priority = MLX5_FLOW_PRIO_RSVD,
6824                         .ingress = 1,
6825                 };
6826                 struct rte_flow_item items[] = {
6827                         [0] = {
6828                                 .type = RTE_FLOW_ITEM_TYPE_END,
6829                         },
6830                 };
6831                 struct rte_flow_action actions[] = {
6832                         [0] = {
6833                                 .type = (enum rte_flow_action_type)
6834                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6835                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6836                                         .src = REG_C_1,
6837                                         .dst = idx,
6838                                 },
6839                         },
6840                         [1] = {
6841                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6842                                 .conf = &(struct rte_flow_action_jump){
6843                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6844                                 },
6845                         },
6846                         [2] = {
6847                                 .type = RTE_FLOW_ACTION_TYPE_END,
6848                         },
6849                 };
6850                 uint32_t flow_idx;
6851                 struct rte_flow *flow;
6852                 struct rte_flow_error error;
6853
6854                 if (!config->dv_flow_en)
6855                         break;
6856                 /* Create internal flow, validation skips copy action. */
6857                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6858                                             actions, false, &error);
6859                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6860                                       flow_idx);
6861                 if (!flow)
6862                         continue;
6863                 config->flow_mreg_c[n++] = idx;
6864                 flow_list_destroy(dev, NULL, flow_idx);
6865         }
6866         for (; n < MLX5_MREG_C_NUM; ++n)
6867                 config->flow_mreg_c[n] = REG_NON;
6868         return 0;
6869 }
6870
6871 /**
6872  * Dump flow raw hw data to file
6873  *
6874  * @param[in] dev
6875  *    The pointer to Ethernet device.
6876  * @param[in] file
6877  *   A pointer to a file for output.
6878  * @param[out] error
6879  *   Perform verbose error reporting if not NULL. PMDs initialize this
6880  *   structure in case of error only.
6881  * @return
6882  *   0 on success, a nagative value otherwise.
6883  */
6884 int
6885 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6886                    FILE *file,
6887                    struct rte_flow_error *error __rte_unused)
6888 {
6889         struct mlx5_priv *priv = dev->data->dev_private;
6890         struct mlx5_dev_ctx_shared *sh = priv->sh;
6891
6892         if (!priv->config.dv_flow_en) {
6893                 if (fputs("device dv flow disabled\n", file) <= 0)
6894                         return -errno;
6895                 return -ENOTSUP;
6896         }
6897         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6898                                        sh->tx_domain, file);
6899 }
6900
6901 /**
6902  * Get aged-out flows.
6903  *
6904  * @param[in] dev
6905  *   Pointer to the Ethernet device structure.
6906  * @param[in] context
6907  *   The address of an array of pointers to the aged-out flows contexts.
6908  * @param[in] nb_countexts
6909  *   The length of context array pointers.
6910  * @param[out] error
6911  *   Perform verbose error reporting if not NULL. Initialized in case of
6912  *   error only.
6913  *
6914  * @return
6915  *   how many contexts get in success, otherwise negative errno value.
6916  *   if nb_contexts is 0, return the amount of all aged contexts.
6917  *   if nb_contexts is not 0 , return the amount of aged flows reported
6918  *   in the context array.
6919  */
6920 int
6921 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6922                         uint32_t nb_contexts, struct rte_flow_error *error)
6923 {
6924         const struct mlx5_flow_driver_ops *fops;
6925         struct rte_flow_attr attr = { .transfer = 0 };
6926
6927         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6928                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6929                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6930                                                     error);
6931         }
6932         DRV_LOG(ERR,
6933                 "port %u get aged flows is not supported.",
6934                  dev->data->port_id);
6935         return -ENOTSUP;
6936 }
6937
6938 /* Wrapper for driver action_validate op callback */
6939 static int
6940 flow_drv_action_validate(struct rte_eth_dev *dev,
6941                          const struct rte_flow_shared_action_conf *conf,
6942                          const struct rte_flow_action *action,
6943                          const struct mlx5_flow_driver_ops *fops,
6944                          struct rte_flow_error *error)
6945 {
6946         static const char err_msg[] = "shared action validation unsupported";
6947
6948         if (!fops->action_validate) {
6949                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6950                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6951                                    NULL, err_msg);
6952                 return -rte_errno;
6953         }
6954         return fops->action_validate(dev, conf, action, error);
6955 }
6956
6957 /**
6958  * Destroys the shared action by handle.
6959  *
6960  * @param dev
6961  *   Pointer to Ethernet device structure.
6962  * @param[in] action
6963  *   Handle for the shared action to be destroyed.
6964  * @param[out] error
6965  *   Perform verbose error reporting if not NULL. PMDs initialize this
6966  *   structure in case of error only.
6967  *
6968  * @return
6969  *   0 on success, a negative errno value otherwise and rte_errno is set.
6970  *
6971  * @note: wrapper for driver action_create op callback.
6972  */
6973 static int
6974 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
6975                            struct rte_flow_shared_action *action,
6976                            struct rte_flow_error *error)
6977 {
6978         static const char err_msg[] = "shared action destruction unsupported";
6979         struct rte_flow_attr attr = { .transfer = 0 };
6980         const struct mlx5_flow_driver_ops *fops =
6981                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
6982
6983         if (!fops->action_destroy) {
6984                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
6985                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
6986                                    NULL, err_msg);
6987                 return -rte_errno;
6988         }
6989         return fops->action_destroy(dev, action, error);
6990 }
6991
6992 /* Wrapper for driver action_destroy op callback */
6993 static int
6994 flow_drv_action_update(struct rte_eth_dev *dev,
6995                        struct rte_flow_shared_action *action,
6996                        const void *action_conf,
6997                        const struct mlx5_flow_driver_ops *fops,
6998                        struct rte_flow_error *error)
6999 {
7000         static const char err_msg[] = "shared action update unsupported";
7001
7002         if (!fops->action_update) {
7003                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7004                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7005                                    NULL, err_msg);
7006                 return -rte_errno;
7007         }
7008         return fops->action_update(dev, action, action_conf, error);
7009 }
7010
7011 /* Wrapper for driver action_destroy op callback */
7012 static int
7013 flow_drv_action_query(struct rte_eth_dev *dev,
7014                       const struct rte_flow_shared_action *action,
7015                       void *data,
7016                       const struct mlx5_flow_driver_ops *fops,
7017                       struct rte_flow_error *error)
7018 {
7019         static const char err_msg[] = "shared action query unsupported";
7020
7021         if (!fops->action_query) {
7022                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7023                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7024                                    NULL, err_msg);
7025                 return -rte_errno;
7026         }
7027         return fops->action_query(dev, action, data, error);
7028 }
7029
7030 /**
7031  * Create shared action for reuse in multiple flow rules.
7032  *
7033  * @param dev
7034  *   Pointer to Ethernet device structure.
7035  * @param[in] action
7036  *   Action configuration for shared action creation.
7037  * @param[out] error
7038  *   Perform verbose error reporting if not NULL. PMDs initialize this
7039  *   structure in case of error only.
7040  * @return
7041  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7042  */
7043 static struct rte_flow_shared_action *
7044 mlx5_shared_action_create(struct rte_eth_dev *dev,
7045                           const struct rte_flow_shared_action_conf *conf,
7046                           const struct rte_flow_action *action,
7047                           struct rte_flow_error *error)
7048 {
7049         static const char err_msg[] = "shared action creation unsupported";
7050         struct rte_flow_attr attr = { .transfer = 0 };
7051         const struct mlx5_flow_driver_ops *fops =
7052                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7053
7054         if (flow_drv_action_validate(dev, conf, action, fops, error))
7055                 return NULL;
7056         if (!fops->action_create) {
7057                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7058                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7059                                    NULL, err_msg);
7060                 return NULL;
7061         }
7062         return fops->action_create(dev, conf, action, error);
7063 }
7064
7065 /**
7066  * Updates inplace the shared action configuration pointed by *action* handle
7067  * with the configuration provided as *action* argument.
7068  * The update of the shared action configuration effects all flow rules reusing
7069  * the action via handle.
7070  *
7071  * @param dev
7072  *   Pointer to Ethernet device structure.
7073  * @param[in] shared_action
7074  *   Handle for the shared action to be updated.
7075  * @param[in] action
7076  *   Action specification used to modify the action pointed by handle.
7077  *   *action* should be of same type with the action pointed by the *action*
7078  *   handle argument, otherwise considered as invalid.
7079  * @param[out] error
7080  *   Perform verbose error reporting if not NULL. PMDs initialize this
7081  *   structure in case of error only.
7082  *
7083  * @return
7084  *   0 on success, a negative errno value otherwise and rte_errno is set.
7085  */
7086 static int
7087 mlx5_shared_action_update(struct rte_eth_dev *dev,
7088                 struct rte_flow_shared_action *shared_action,
7089                 const struct rte_flow_action *action,
7090                 struct rte_flow_error *error)
7091 {
7092         struct rte_flow_attr attr = { .transfer = 0 };
7093         const struct mlx5_flow_driver_ops *fops =
7094                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7095         int ret;
7096
7097         ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7098         if (ret)
7099                 return ret;
7100         return flow_drv_action_update(dev, shared_action, action->conf, fops,
7101                                       error);
7102 }
7103
7104 /**
7105  * Query the shared action by handle.
7106  *
7107  * This function allows retrieving action-specific data such as counters.
7108  * Data is gathered by special action which may be present/referenced in
7109  * more than one flow rule definition.
7110  *
7111  * \see RTE_FLOW_ACTION_TYPE_COUNT
7112  *
7113  * @param dev
7114  *   Pointer to Ethernet device structure.
7115  * @param[in] action
7116  *   Handle for the shared action to query.
7117  * @param[in, out] data
7118  *   Pointer to storage for the associated query data type.
7119  * @param[out] error
7120  *   Perform verbose error reporting if not NULL. PMDs initialize this
7121  *   structure in case of error only.
7122  *
7123  * @return
7124  *   0 on success, a negative errno value otherwise and rte_errno is set.
7125  */
7126 static int
7127 mlx5_shared_action_query(struct rte_eth_dev *dev,
7128                          const struct rte_flow_shared_action *action,
7129                          void *data,
7130                          struct rte_flow_error *error)
7131 {
7132         struct rte_flow_attr attr = { .transfer = 0 };
7133         const struct mlx5_flow_driver_ops *fops =
7134                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7135
7136         return flow_drv_action_query(dev, action, data, fops, error);
7137 }
7138
7139 /**
7140  * Destroy all shared actions.
7141  *
7142  * @param dev
7143  *   Pointer to Ethernet device.
7144  *
7145  * @return
7146  *   0 on success, a negative errno value otherwise and rte_errno is set.
7147  */
7148 int
7149 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7150 {
7151         struct rte_flow_error error;
7152         struct mlx5_priv *priv = dev->data->dev_private;
7153         struct mlx5_shared_action_rss *action;
7154         int ret = 0;
7155         uint32_t idx;
7156
7157         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7158                       priv->rss_shared_actions, idx, action, next) {
7159                 ret |= mlx5_shared_action_destroy(dev,
7160                        (struct rte_flow_shared_action *)(uintptr_t)idx, &error);
7161         }
7162         return ret;
7163 }
7164
7165 #ifndef HAVE_MLX5DV_DR
7166 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
7167 #else
7168 #define MLX5_DOMAIN_SYNC_FLOW \
7169         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
7170 #endif
7171
7172 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
7173 {
7174         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
7175         const struct mlx5_flow_driver_ops *fops;
7176         int ret;
7177         struct rte_flow_attr attr = { .transfer = 0 };
7178
7179         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7180         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
7181         if (ret > 0)
7182                 ret = -ret;
7183         return ret;
7184 }
7185
7186 /**
7187  * tunnel offload functionalilty is defined for DV environment only
7188  */
7189 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7190 __extension__
7191 union tunnel_offload_mark {
7192         uint32_t val;
7193         struct {
7194                 uint32_t app_reserve:8;
7195                 uint32_t table_id:15;
7196                 uint32_t transfer:1;
7197                 uint32_t _unused_:8;
7198         };
7199 };
7200
7201 static bool
7202 mlx5_access_tunnel_offload_db
7203         (struct rte_eth_dev *dev,
7204          bool (*match)(struct rte_eth_dev *,
7205                        struct mlx5_flow_tunnel *, const void *),
7206          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7207          void (*miss)(struct rte_eth_dev *, void *),
7208          void *ctx, bool lock_op);
7209
7210 static int
7211 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
7212                              struct rte_flow *flow,
7213                              const struct rte_flow_attr *attr,
7214                              const struct rte_flow_action *app_actions,
7215                              uint32_t flow_idx,
7216                              struct tunnel_default_miss_ctx *ctx,
7217                              struct rte_flow_error *error)
7218 {
7219         struct mlx5_priv *priv = dev->data->dev_private;
7220         struct mlx5_flow *dev_flow;
7221         struct rte_flow_attr miss_attr = *attr;
7222         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
7223         const struct rte_flow_item miss_items[2] = {
7224                 {
7225                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7226                         .spec = NULL,
7227                         .last = NULL,
7228                         .mask = NULL
7229                 },
7230                 {
7231                         .type = RTE_FLOW_ITEM_TYPE_END,
7232                         .spec = NULL,
7233                         .last = NULL,
7234                         .mask = NULL
7235                 }
7236         };
7237         union tunnel_offload_mark mark_id;
7238         struct rte_flow_action_mark miss_mark;
7239         struct rte_flow_action miss_actions[3] = {
7240                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
7241                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
7242         };
7243         const struct rte_flow_action_jump *jump_data;
7244         uint32_t i, flow_table = 0; /* prevent compilation warning */
7245         struct flow_grp_info grp_info = {
7246                 .external = 1,
7247                 .transfer = attr->transfer,
7248                 .fdb_def_rule = !!priv->fdb_def_rule,
7249                 .std_tbl_fix = 0,
7250         };
7251         int ret;
7252
7253         if (!attr->transfer) {
7254                 uint32_t q_size;
7255
7256                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
7257                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
7258                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
7259                                          0, SOCKET_ID_ANY);
7260                 if (!ctx->queue)
7261                         return rte_flow_error_set
7262                                 (error, ENOMEM,
7263                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7264                                 NULL, "invalid default miss RSS");
7265                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7266                 ctx->action_rss.level = 0,
7267                 ctx->action_rss.types = priv->rss_conf.rss_hf,
7268                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
7269                 ctx->action_rss.queue_num = priv->reta_idx_n,
7270                 ctx->action_rss.key = priv->rss_conf.rss_key,
7271                 ctx->action_rss.queue = ctx->queue;
7272                 if (!priv->reta_idx_n || !priv->rxqs_n)
7273                         return rte_flow_error_set
7274                                 (error, EINVAL,
7275                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7276                                 NULL, "invalid port configuration");
7277                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
7278                         ctx->action_rss.types = 0;
7279                 for (i = 0; i != priv->reta_idx_n; ++i)
7280                         ctx->queue[i] = (*priv->reta_idx)[i];
7281         } else {
7282                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
7283                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
7284         }
7285         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
7286         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
7287         jump_data = app_actions->conf;
7288         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
7289         miss_attr.group = jump_data->group;
7290         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
7291                                        &flow_table, &grp_info, error);
7292         if (ret)
7293                 return rte_flow_error_set(error, EINVAL,
7294                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
7295                                           NULL, "invalid tunnel id");
7296         mark_id.app_reserve = 0;
7297         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
7298         mark_id.transfer = !!attr->transfer;
7299         mark_id._unused_ = 0;
7300         miss_mark.id = mark_id.val;
7301         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
7302                                     miss_items, miss_actions, flow_idx, error);
7303         if (!dev_flow)
7304                 return -rte_errno;
7305         dev_flow->flow = flow;
7306         dev_flow->external = true;
7307         dev_flow->tunnel = tunnel;
7308         /* Subflow object was created, we must include one in the list. */
7309         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7310                       dev_flow->handle, next);
7311         DRV_LOG(DEBUG,
7312                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
7313                 dev->data->port_id, tunnel->app_tunnel.type,
7314                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
7315         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
7316                                   miss_actions, error);
7317         if (!ret)
7318                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
7319                                                   error);
7320
7321         return ret;
7322 }
7323
7324 static const struct mlx5_flow_tbl_data_entry  *
7325 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7326 {
7327         struct mlx5_priv *priv = dev->data->dev_private;
7328         struct mlx5_dev_ctx_shared *sh = priv->sh;
7329         struct mlx5_hlist_entry *he;
7330         union tunnel_offload_mark mbits = { .val = mark };
7331         union mlx5_flow_tbl_key table_key = {
7332                 {
7333                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7334                         .dummy = 0,
7335                         .domain = !!mbits.transfer,
7336                         .direction = 0,
7337                 }
7338         };
7339         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
7340         return he ?
7341                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7342 }
7343
7344 static void
7345 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
7346                                    struct mlx5_hlist_entry *entry)
7347 {
7348         struct mlx5_dev_ctx_shared *sh = list->ctx;
7349         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
7350
7351         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7352                         tunnel_flow_tbl_to_id(tte->flow_table));
7353         mlx5_free(tte);
7354 }
7355
7356 static struct mlx5_hlist_entry *
7357 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
7358                                    uint64_t key __rte_unused,
7359                                    void *ctx __rte_unused)
7360 {
7361         struct mlx5_dev_ctx_shared *sh = list->ctx;
7362         struct tunnel_tbl_entry *tte;
7363
7364         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7365                           sizeof(*tte), 0,
7366                           SOCKET_ID_ANY);
7367         if (!tte)
7368                 goto err;
7369         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7370                           &tte->flow_table);
7371         if (tte->flow_table >= MLX5_MAX_TABLES) {
7372                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
7373                         tte->flow_table);
7374                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
7375                                 tte->flow_table);
7376                 goto err;
7377         } else if (!tte->flow_table) {
7378                 goto err;
7379         }
7380         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7381         return &tte->hash;
7382 err:
7383         if (tte)
7384                 mlx5_free(tte);
7385         return NULL;
7386 }
7387
7388 static uint32_t
7389 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7390                                 const struct mlx5_flow_tunnel *tunnel,
7391                                 uint32_t group, uint32_t *table,
7392                                 struct rte_flow_error *error)
7393 {
7394         struct mlx5_hlist_entry *he;
7395         struct tunnel_tbl_entry *tte;
7396         union tunnel_tbl_key key = {
7397                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7398                 .group = group
7399         };
7400         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7401         struct mlx5_hlist *group_hash;
7402
7403         group_hash = tunnel ? tunnel->groups : thub->groups;
7404         he = mlx5_hlist_register(group_hash, key.val, NULL);
7405         if (!he)
7406                 return rte_flow_error_set(error, EINVAL,
7407                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7408                                           NULL,
7409                                           "tunnel group index not supported");
7410         tte = container_of(he, typeof(*tte), hash);
7411         *table = tte->flow_table;
7412         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7413                 dev->data->port_id, key.tunnel_id, group, *table);
7414         return 0;
7415 }
7416
7417 static void
7418 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
7419                       struct mlx5_flow_tunnel *tunnel)
7420 {
7421         struct mlx5_priv *priv = dev->data->dev_private;
7422         struct mlx5_indexed_pool *ipool;
7423
7424         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
7425                 dev->data->port_id, tunnel->tunnel_id);
7426         mlx5_hlist_destroy(tunnel->groups);
7427         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7428         mlx5_ipool_free(ipool, tunnel->tunnel_id);
7429 }
7430
7431 static bool
7432 mlx5_access_tunnel_offload_db
7433         (struct rte_eth_dev *dev,
7434          bool (*match)(struct rte_eth_dev *,
7435                        struct mlx5_flow_tunnel *, const void *),
7436          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
7437          void (*miss)(struct rte_eth_dev *, void *),
7438          void *ctx, bool lock_op)
7439 {
7440         bool verdict = false;
7441         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7442         struct mlx5_flow_tunnel *tunnel;
7443
7444         rte_spinlock_lock(&thub->sl);
7445         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
7446                 verdict = match(dev, tunnel, (const void *)ctx);
7447                 if (verdict)
7448                         break;
7449         }
7450         if (!lock_op)
7451                 rte_spinlock_unlock(&thub->sl);
7452         if (verdict && hit)
7453                 hit(dev, tunnel, ctx);
7454         if (!verdict && miss)
7455                 miss(dev, ctx);
7456         if (lock_op)
7457                 rte_spinlock_unlock(&thub->sl);
7458
7459         return verdict;
7460 }
7461
7462 struct tunnel_db_find_tunnel_id_ctx {
7463         uint32_t tunnel_id;
7464         struct mlx5_flow_tunnel *tunnel;
7465 };
7466
7467 static bool
7468 find_tunnel_id_match(struct rte_eth_dev *dev,
7469                      struct mlx5_flow_tunnel *tunnel, const void *x)
7470 {
7471         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7472
7473         RTE_SET_USED(dev);
7474         return tunnel->tunnel_id == ctx->tunnel_id;
7475 }
7476
7477 static void
7478 find_tunnel_id_hit(struct rte_eth_dev *dev,
7479                    struct mlx5_flow_tunnel *tunnel, void *x)
7480 {
7481         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
7482         RTE_SET_USED(dev);
7483         ctx->tunnel = tunnel;
7484 }
7485
7486 static struct mlx5_flow_tunnel *
7487 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
7488 {
7489         struct tunnel_db_find_tunnel_id_ctx ctx = {
7490                 .tunnel_id = id,
7491         };
7492
7493         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
7494                                       find_tunnel_id_hit, NULL, &ctx, true);
7495
7496         return ctx.tunnel;
7497 }
7498
7499 static struct mlx5_flow_tunnel *
7500 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
7501                           const struct rte_flow_tunnel *app_tunnel)
7502 {
7503         struct mlx5_priv *priv = dev->data->dev_private;
7504         struct mlx5_indexed_pool *ipool;
7505         struct mlx5_flow_tunnel *tunnel;
7506         uint32_t id;
7507
7508         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
7509         tunnel = mlx5_ipool_zmalloc(ipool, &id);
7510         if (!tunnel)
7511                 return NULL;
7512         if (id >= MLX5_MAX_TUNNELS) {
7513                 mlx5_ipool_free(ipool, id);
7514                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
7515                 return NULL;
7516         }
7517         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
7518                                            mlx5_flow_tunnel_grp2tbl_create_cb,
7519                                            NULL,
7520                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
7521         if (!tunnel->groups) {
7522                 mlx5_ipool_free(ipool, id);
7523                 return NULL;
7524         }
7525         tunnel->groups->ctx = priv->sh;
7526         /* initiate new PMD tunnel */
7527         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
7528         tunnel->tunnel_id = id;
7529         tunnel->action.type = (typeof(tunnel->action.type))
7530                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
7531         tunnel->action.conf = tunnel;
7532         tunnel->item.type = (typeof(tunnel->item.type))
7533                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
7534         tunnel->item.spec = tunnel;
7535         tunnel->item.last = NULL;
7536         tunnel->item.mask = NULL;
7537
7538         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
7539                 dev->data->port_id, tunnel->tunnel_id);
7540
7541         return tunnel;
7542 }
7543
7544 struct tunnel_db_get_tunnel_ctx {
7545         const struct rte_flow_tunnel *app_tunnel;
7546         struct mlx5_flow_tunnel *tunnel;
7547 };
7548
7549 static bool get_tunnel_match(struct rte_eth_dev *dev,
7550                              struct mlx5_flow_tunnel *tunnel, const void *x)
7551 {
7552         const struct tunnel_db_get_tunnel_ctx *ctx = x;
7553
7554         RTE_SET_USED(dev);
7555         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
7556                        sizeof(*ctx->app_tunnel));
7557 }
7558
7559 static void get_tunnel_hit(struct rte_eth_dev *dev,
7560                            struct mlx5_flow_tunnel *tunnel, void *x)
7561 {
7562         /* called under tunnel spinlock protection */
7563         struct tunnel_db_get_tunnel_ctx *ctx = x;
7564
7565         RTE_SET_USED(dev);
7566         tunnel->refctn++;
7567         ctx->tunnel = tunnel;
7568 }
7569
7570 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
7571 {
7572         /* called under tunnel spinlock protection */
7573         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7574         struct tunnel_db_get_tunnel_ctx *ctx = x;
7575
7576         rte_spinlock_unlock(&thub->sl);
7577         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
7578         ctx->tunnel->refctn = 1;
7579         rte_spinlock_lock(&thub->sl);
7580         if (ctx->tunnel)
7581                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
7582 }
7583
7584
7585 static int
7586 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
7587                      const struct rte_flow_tunnel *app_tunnel,
7588                      struct mlx5_flow_tunnel **tunnel)
7589 {
7590         struct tunnel_db_get_tunnel_ctx ctx = {
7591                 .app_tunnel = app_tunnel,
7592         };
7593
7594         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
7595                                       get_tunnel_miss, &ctx, true);
7596         *tunnel = ctx.tunnel;
7597         return ctx.tunnel ? 0 : -ENOMEM;
7598 }
7599
7600 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
7601 {
7602         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
7603
7604         if (!thub)
7605                 return;
7606         if (!LIST_EMPTY(&thub->tunnels))
7607                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
7608         mlx5_hlist_destroy(thub->groups);
7609         mlx5_free(thub);
7610 }
7611
7612 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
7613 {
7614         int err;
7615         struct mlx5_flow_tunnel_hub *thub;
7616
7617         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
7618                            0, SOCKET_ID_ANY);
7619         if (!thub)
7620                 return -ENOMEM;
7621         LIST_INIT(&thub->tunnels);
7622         rte_spinlock_init(&thub->sl);
7623         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
7624                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
7625                                          NULL,
7626                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
7627         if (!thub->groups) {
7628                 err = -rte_errno;
7629                 goto err;
7630         }
7631         thub->groups->ctx = sh;
7632         sh->tunnel_hub = thub;
7633
7634         return 0;
7635
7636 err:
7637         if (thub->groups)
7638                 mlx5_hlist_destroy(thub->groups);
7639         if (thub)
7640                 mlx5_free(thub);
7641         return err;
7642 }
7643
7644 static inline bool
7645 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
7646                           struct rte_flow_tunnel *tunnel,
7647                           const char *err_msg)
7648 {
7649         err_msg = NULL;
7650         if (!is_tunnel_offload_active(dev)) {
7651                 err_msg = "tunnel offload was not activated";
7652                 goto out;
7653         } else if (!tunnel) {
7654                 err_msg = "no application tunnel";
7655                 goto out;
7656         }
7657
7658         switch (tunnel->type) {
7659         default:
7660                 err_msg = "unsupported tunnel type";
7661                 goto out;
7662         case RTE_FLOW_ITEM_TYPE_VXLAN:
7663                 break;
7664         }
7665
7666 out:
7667         return !err_msg;
7668 }
7669
7670 static int
7671 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
7672                     struct rte_flow_tunnel *app_tunnel,
7673                     struct rte_flow_action **actions,
7674                     uint32_t *num_of_actions,
7675                     struct rte_flow_error *error)
7676 {
7677         int ret;
7678         struct mlx5_flow_tunnel *tunnel;
7679         const char *err_msg = NULL;
7680         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7681
7682         if (!verdict)
7683                 return rte_flow_error_set(error, EINVAL,
7684                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7685                                           err_msg);
7686         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7687         if (ret < 0) {
7688                 return rte_flow_error_set(error, ret,
7689                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
7690                                           "failed to initialize pmd tunnel");
7691         }
7692         *actions = &tunnel->action;
7693         *num_of_actions = 1;
7694         return 0;
7695 }
7696
7697 static int
7698 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
7699                        struct rte_flow_tunnel *app_tunnel,
7700                        struct rte_flow_item **items,
7701                        uint32_t *num_of_items,
7702                        struct rte_flow_error *error)
7703 {
7704         int ret;
7705         struct mlx5_flow_tunnel *tunnel;
7706         const char *err_msg = NULL;
7707         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
7708
7709         if (!verdict)
7710                 return rte_flow_error_set(error, EINVAL,
7711                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7712                                           err_msg);
7713         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
7714         if (ret < 0) {
7715                 return rte_flow_error_set(error, ret,
7716                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7717                                           "failed to initialize pmd tunnel");
7718         }
7719         *items = &tunnel->item;
7720         *num_of_items = 1;
7721         return 0;
7722 }
7723
7724 struct tunnel_db_element_release_ctx {
7725         struct rte_flow_item *items;
7726         struct rte_flow_action *actions;
7727         uint32_t num_elements;
7728         struct rte_flow_error *error;
7729         int ret;
7730 };
7731
7732 static bool
7733 tunnel_element_release_match(struct rte_eth_dev *dev,
7734                              struct mlx5_flow_tunnel *tunnel, const void *x)
7735 {
7736         const struct tunnel_db_element_release_ctx *ctx = x;
7737
7738         RTE_SET_USED(dev);
7739         if (ctx->num_elements != 1)
7740                 return false;
7741         else if (ctx->items)
7742                 return ctx->items == &tunnel->item;
7743         else if (ctx->actions)
7744                 return ctx->actions == &tunnel->action;
7745
7746         return false;
7747 }
7748
7749 static void
7750 tunnel_element_release_hit(struct rte_eth_dev *dev,
7751                            struct mlx5_flow_tunnel *tunnel, void *x)
7752 {
7753         struct tunnel_db_element_release_ctx *ctx = x;
7754         ctx->ret = 0;
7755         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7756                 mlx5_flow_tunnel_free(dev, tunnel);
7757 }
7758
7759 static void
7760 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
7761 {
7762         struct tunnel_db_element_release_ctx *ctx = x;
7763         RTE_SET_USED(dev);
7764         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
7765                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
7766                                       "invalid argument");
7767 }
7768
7769 static int
7770 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
7771                        struct rte_flow_item *pmd_items,
7772                        uint32_t num_items, struct rte_flow_error *err)
7773 {
7774         struct tunnel_db_element_release_ctx ctx = {
7775                 .items = pmd_items,
7776                 .actions = NULL,
7777                 .num_elements = num_items,
7778                 .error = err,
7779         };
7780
7781         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7782                                       tunnel_element_release_hit,
7783                                       tunnel_element_release_miss, &ctx, false);
7784
7785         return ctx.ret;
7786 }
7787
7788 static int
7789 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
7790                          struct rte_flow_action *pmd_actions,
7791                          uint32_t num_actions, struct rte_flow_error *err)
7792 {
7793         struct tunnel_db_element_release_ctx ctx = {
7794                 .items = NULL,
7795                 .actions = pmd_actions,
7796                 .num_elements = num_actions,
7797                 .error = err,
7798         };
7799
7800         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
7801                                       tunnel_element_release_hit,
7802                                       tunnel_element_release_miss, &ctx, false);
7803
7804         return ctx.ret;
7805 }
7806
7807 static int
7808 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
7809                                   struct rte_mbuf *m,
7810                                   struct rte_flow_restore_info *info,
7811                                   struct rte_flow_error *err)
7812 {
7813         uint64_t ol_flags = m->ol_flags;
7814         const struct mlx5_flow_tbl_data_entry *tble;
7815         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
7816
7817         if (!is_tunnel_offload_active(dev)) {
7818                 info->flags = 0;
7819                 return 0;
7820         }
7821
7822         if ((ol_flags & mask) != mask)
7823                 goto err;
7824         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
7825         if (!tble) {
7826                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
7827                         dev->data->port_id, m->hash.fdir.hi);
7828                 goto err;
7829         }
7830         MLX5_ASSERT(tble->tunnel);
7831         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
7832         info->group_id = tble->group_id;
7833         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
7834                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
7835                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
7836
7837         return 0;
7838
7839 err:
7840         return rte_flow_error_set(err, EINVAL,
7841                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7842                                   "failed to get restore info");
7843 }
7844
7845 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
7846 static int
7847 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
7848                            __rte_unused struct rte_flow_tunnel *app_tunnel,
7849                            __rte_unused struct rte_flow_action **actions,
7850                            __rte_unused uint32_t *num_of_actions,
7851                            __rte_unused struct rte_flow_error *error)
7852 {
7853         return -ENOTSUP;
7854 }
7855
7856 static int
7857 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
7858                        __rte_unused struct rte_flow_tunnel *app_tunnel,
7859                        __rte_unused struct rte_flow_item **items,
7860                        __rte_unused uint32_t *num_of_items,
7861                        __rte_unused struct rte_flow_error *error)
7862 {
7863         return -ENOTSUP;
7864 }
7865
7866 static int
7867 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
7868                               __rte_unused struct rte_flow_item *pmd_items,
7869                               __rte_unused uint32_t num_items,
7870                               __rte_unused struct rte_flow_error *err)
7871 {
7872         return -ENOTSUP;
7873 }
7874
7875 static int
7876 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
7877                                 __rte_unused struct rte_flow_action *pmd_action,
7878                                 __rte_unused uint32_t num_actions,
7879                                 __rte_unused struct rte_flow_error *err)
7880 {
7881         return -ENOTSUP;
7882 }
7883
7884 static int
7885 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
7886                                   __rte_unused struct rte_mbuf *m,
7887                                   __rte_unused struct rte_flow_restore_info *i,
7888                                   __rte_unused struct rte_flow_error *err)
7889 {
7890         return -ENOTSUP;
7891 }
7892
7893 static int
7894 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
7895                              __rte_unused struct rte_flow *flow,
7896                              __rte_unused const struct rte_flow_attr *attr,
7897                              __rte_unused const struct rte_flow_action *actions,
7898                              __rte_unused uint32_t flow_idx,
7899                              __rte_unused struct tunnel_default_miss_ctx *ctx,
7900                              __rte_unused struct rte_flow_error *error)
7901 {
7902         return -ENOTSUP;
7903 }
7904
7905 static struct mlx5_flow_tunnel *
7906 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
7907                     __rte_unused uint32_t id)
7908 {
7909         return NULL;
7910 }
7911
7912 static void
7913 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
7914                       __rte_unused struct mlx5_flow_tunnel *tunnel)
7915 {
7916 }
7917
7918 static uint32_t
7919 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
7920                                 __rte_unused const struct mlx5_flow_tunnel *t,
7921                                 __rte_unused uint32_t group,
7922                                 __rte_unused uint32_t *table,
7923                                 struct rte_flow_error *error)
7924 {
7925         return rte_flow_error_set(error, ENOTSUP,
7926                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7927                                   "tunnel offload requires DV support");
7928 }
7929
7930 void
7931 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
7932                         __rte_unused  uint16_t port_id)
7933 {
7934 }
7935 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
7936