drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 static struct mlx5_flow_tunnel *
  37 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  38 static void
  39 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  40 static const struct mlx5_flow_tbl_data_entry  *
  41 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark);
  42 static int
  43 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
  44                      const struct rte_flow_tunnel *app_tunnel,
  45                      struct mlx5_flow_tunnel **tunnel);
  46
  47
  48 /** Device flow drivers. */
  49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  50
  51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  52
  53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  54         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  56         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  57 #endif
  58         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  59         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  60 };
  61
  62 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  63 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  64         (const int []){ \
  65                 __VA_ARGS__, 0, \
  66         }
  67
  68 /** Node object of input graph for mlx5_flow_expand_rss(). */
  69 struct mlx5_flow_expand_node {
  70         const int *const next;
  71         /**<
  72          * List of next node indexes. Index 0 is interpreted as a terminator.
  73          */
  74         const enum rte_flow_item_type type;
  75         /**< Pattern item type of current node. */
  76         uint64_t rss_types;
  77         /**<
  78          * RSS types bit-field associated with this node
  79          * (see ETH_RSS_* definitions).
  80          */
  81 };
  82
  83 /** Object returned by mlx5_flow_expand_rss(). */
  84 struct mlx5_flow_expand_rss {
  85         uint32_t entries;
  86         /**< Number of entries @p patterns and @p priorities. */
  87         struct {
  88                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
  89                 uint32_t priority; /**< Priority offset for each expansion. */
  90         } entry[];
  91 };
  92
  93 static enum rte_flow_item_type
  94 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
  95 {
  96         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
  97         uint16_t ether_type = 0;
  98         uint16_t ether_type_m;
  99         uint8_t ip_next_proto = 0;
 100         uint8_t ip_next_proto_m;
 101
 102         if (item == NULL || item->spec == NULL)
 103                 return ret;
 104         switch (item->type) {
 105         case RTE_FLOW_ITEM_TYPE_ETH:
 106                 if (item->mask)
 107                         ether_type_m = ((const struct rte_flow_item_eth *)
 108                                                 (item->mask))->type;
 109                 else
 110                         ether_type_m = rte_flow_item_eth_mask.type;
 111                 if (ether_type_m != RTE_BE16(0xFFFF))
 112                         break;
 113                 ether_type = ((const struct rte_flow_item_eth *)
 114                                 (item->spec))->type;
 115                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 116                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 117                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 118                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 119                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 120                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 121                 else
 122                         ret = RTE_FLOW_ITEM_TYPE_END;
 123                 break;
 124         case RTE_FLOW_ITEM_TYPE_VLAN:
 125                 if (item->mask)
 126                         ether_type_m = ((const struct rte_flow_item_vlan *)
 127                                                 (item->mask))->inner_type;
 128                 else
 129                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 130                 if (ether_type_m != RTE_BE16(0xFFFF))
 131                         break;
 132                 ether_type = ((const struct rte_flow_item_vlan *)
 133                                 (item->spec))->inner_type;
 134                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 135                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 136                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 137                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 138                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 139                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 140                 else
 141                         ret = RTE_FLOW_ITEM_TYPE_END;
 142                 break;
 143         case RTE_FLOW_ITEM_TYPE_IPV4:
 144                 if (item->mask)
 145                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 146                                         (item->mask))->hdr.next_proto_id;
 147                 else
 148                         ip_next_proto_m =
 149                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 150                 if (ip_next_proto_m != 0xFF)
 151                         break;
 152                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 153                                 (item->spec))->hdr.next_proto_id;
 154                 if (ip_next_proto == IPPROTO_UDP)
 155                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 156                 else if (ip_next_proto == IPPROTO_TCP)
 157                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 158                 else if (ip_next_proto == IPPROTO_IP)
 159                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 160                 else if (ip_next_proto == IPPROTO_IPV6)
 161                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 162                 else
 163                         ret = RTE_FLOW_ITEM_TYPE_END;
 164                 break;
 165         case RTE_FLOW_ITEM_TYPE_IPV6:
 166                 if (item->mask)
 167                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 168                                                 (item->mask))->hdr.proto;
 169                 else
 170                         ip_next_proto_m =
 171                                 rte_flow_item_ipv6_mask.hdr.proto;
 172                 if (ip_next_proto_m != 0xFF)
 173                         break;
 174                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 175                                 (item->spec))->hdr.proto;
 176                 if (ip_next_proto == IPPROTO_UDP)
 177                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 178                 else if (ip_next_proto == IPPROTO_TCP)
 179                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 180                 else if (ip_next_proto == IPPROTO_IP)
 181                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 182                 else if (ip_next_proto == IPPROTO_IPV6)
 183                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 184                 else
 185                         ret = RTE_FLOW_ITEM_TYPE_END;
 186                 break;
 187         default:
 188                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 189                 break;
 190         }
 191         return ret;
 192 }
 193
 194 /**
 195  * Expand RSS flows into several possible flows according to the RSS hash
 196  * fields requested and the driver capabilities.
 197  *
 198  * @param[out] buf
 199  *   Buffer to store the result expansion.
 200  * @param[in] size
 201  *   Buffer size in bytes. If 0, @p buf can be NULL.
 202  * @param[in] pattern
 203  *   User flow pattern.
 204  * @param[in] types
 205  *   RSS types to expand (see ETH_RSS_* definitions).
 206  * @param[in] graph
 207  *   Input graph to expand @p pattern according to @p types.
 208  * @param[in] graph_root_index
 209  *   Index of root node in @p graph, typically 0.
 210  *
 211  * @return
 212  *   A positive value representing the size of @p buf in bytes regardless of
 213  *   @p size on success, a negative errno value otherwise and rte_errno is
 214  *   set, the following errors are defined:
 215  *
 216  *   -E2BIG: graph-depth @p graph is too deep.
 217  */
 218 static int
 219 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 220                      const struct rte_flow_item *pattern, uint64_t types,
 221                      const struct mlx5_flow_expand_node graph[],
 222                      int graph_root_index)
 223 {
 224         const int elt_n = 8;
 225         const struct rte_flow_item *item;
 226         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 227         const int *next_node;
 228         const int *stack[elt_n];
 229         int stack_pos = 0;
 230         struct rte_flow_item flow_items[elt_n];
 231         unsigned int i;
 232         size_t lsize;
 233         size_t user_pattern_size = 0;
 234         void *addr = NULL;
 235         const struct mlx5_flow_expand_node *next = NULL;
 236         struct rte_flow_item missed_item;
 237         int missed = 0;
 238         int elt = 0;
 239         const struct rte_flow_item *last_item = NULL;
 240
 241         memset(&missed_item, 0, sizeof(missed_item));
 242         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 243                 elt_n * sizeof(buf->entry[0]);
 244         if (lsize <= size) {
 245                 buf->entry[0].priority = 0;
 246                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 247                 buf->entries = 0;
 248                 addr = buf->entry[0].pattern;
 249         }
 250         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 251                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 252                         last_item = item;
 253                 for (i = 0; node->next && node->next[i]; ++i) {
 254                         next = &graph[node->next[i]];
 255                         if (next->type == item->type)
 256                                 break;
 257                 }
 258                 if (next)
 259                         node = next;
 260                 user_pattern_size += sizeof(*item);
 261         }
 262         user_pattern_size += sizeof(*item); /* Handle END item. */
 263         lsize += user_pattern_size;
 264         /* Copy the user pattern in the first entry of the buffer. */
 265         if (lsize <= size) {
 266                 rte_memcpy(addr, pattern, user_pattern_size);
 267                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 268                 buf->entries = 1;
 269         }
 270         /* Start expanding. */
 271         memset(flow_items, 0, sizeof(flow_items));
 272         user_pattern_size -= sizeof(*item);
 273         /*
 274          * Check if the last valid item has spec set, need complete pattern,
 275          * and the pattern can be used for expansion.
 276          */
 277         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 278         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 279                 /* Item type END indicates expansion is not required. */
 280                 return lsize;
 281         }
 282         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 283                 next = NULL;
 284                 missed = 1;
 285                 for (i = 0; node->next && node->next[i]; ++i) {
 286                         next = &graph[node->next[i]];
 287                         if (next->type == missed_item.type) {
 288                                 flow_items[0].type = missed_item.type;
 289                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 290                                 break;
 291                         }
 292                         next = NULL;
 293                 }
 294         }
 295         if (next && missed) {
 296                 elt = 2; /* missed item + item end. */
 297                 node = next;
 298                 lsize += elt * sizeof(*item) + user_pattern_size;
 299                 if ((node->rss_types & types) && lsize <= size) {
 300                         buf->entry[buf->entries].priority = 1;
 301                         buf->entry[buf->entries].pattern = addr;
 302                         buf->entries++;
 303                         rte_memcpy(addr, buf->entry[0].pattern,
 304                                    user_pattern_size);
 305                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 306                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 307                         addr = (void *)(((uintptr_t)addr) +
 308                                         elt * sizeof(*item));
 309                 }
 310         }
 311         memset(flow_items, 0, sizeof(flow_items));
 312         next_node = node->next;
 313         stack[stack_pos] = next_node;
 314         node = next_node ? &graph[*next_node] : NULL;
 315         while (node) {
 316                 flow_items[stack_pos].type = node->type;
 317                 if (node->rss_types & types) {
 318                         /*
 319                          * compute the number of items to copy from the
 320                          * expansion and copy it.
 321                          * When the stack_pos is 0, there are 1 element in it,
 322                          * plus the addition END item.
 323                          */
 324                         elt = stack_pos + 2;
 325                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 326                         lsize += elt * sizeof(*item) + user_pattern_size;
 327                         if (lsize <= size) {
 328                                 size_t n = elt * sizeof(*item);
 329
 330                                 buf->entry[buf->entries].priority =
 331                                         stack_pos + 1 + missed;
 332                                 buf->entry[buf->entries].pattern = addr;
 333                                 buf->entries++;
 334                                 rte_memcpy(addr, buf->entry[0].pattern,
 335                                            user_pattern_size);
 336                                 addr = (void *)(((uintptr_t)addr) +
 337                                                 user_pattern_size);
 338                                 rte_memcpy(addr, &missed_item,
 339                                            missed * sizeof(*item));
 340                                 addr = (void *)(((uintptr_t)addr) +
 341                                         missed * sizeof(*item));
 342                                 rte_memcpy(addr, flow_items, n);
 343                                 addr = (void *)(((uintptr_t)addr) + n);
 344                         }
 345                 }
 346                 /* Go deeper. */
 347                 if (node->next) {
 348                         next_node = node->next;
 349                         if (stack_pos++ == elt_n) {
 350                                 rte_errno = E2BIG;
 351                                 return -rte_errno;
 352                         }
 353                         stack[stack_pos] = next_node;
 354                 } else if (*(next_node + 1)) {
 355                         /* Follow up with the next possibility. */
 356                         ++next_node;
 357                 } else {
 358                         /* Move to the next path. */
 359                         if (stack_pos)
 360                                 next_node = stack[--stack_pos];
 361                         next_node++;
 362                         stack[stack_pos] = next_node;
 363                 }
 364                 node = *next_node ? &graph[*next_node] : NULL;
 365         };
 366         /* no expanded flows but we have missed item, create one rule for it */
 367         if (buf->entries == 1 && missed != 0) {
 368                 elt = 2;
 369                 lsize += elt * sizeof(*item) + user_pattern_size;
 370                 if (lsize <= size) {
 371                         buf->entry[buf->entries].priority = 1;
 372                         buf->entry[buf->entries].pattern = addr;
 373                         buf->entries++;
 374                         flow_items[0].type = missed_item.type;
 375                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 376                         rte_memcpy(addr, buf->entry[0].pattern,
 377                                    user_pattern_size);
 378                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 379                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 380                         addr = (void *)(((uintptr_t)addr) +
 381                                         elt * sizeof(*item));
 382                 }
 383         }
 384         return lsize;
 385 }
 386
 387 enum mlx5_expansion {
 388         MLX5_EXPANSION_ROOT,
 389         MLX5_EXPANSION_ROOT_OUTER,
 390         MLX5_EXPANSION_ROOT_ETH_VLAN,
 391         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 392         MLX5_EXPANSION_OUTER_ETH,
 393         MLX5_EXPANSION_OUTER_ETH_VLAN,
 394         MLX5_EXPANSION_OUTER_VLAN,
 395         MLX5_EXPANSION_OUTER_IPV4,
 396         MLX5_EXPANSION_OUTER_IPV4_UDP,
 397         MLX5_EXPANSION_OUTER_IPV4_TCP,
 398         MLX5_EXPANSION_OUTER_IPV6,
 399         MLX5_EXPANSION_OUTER_IPV6_UDP,
 400         MLX5_EXPANSION_OUTER_IPV6_TCP,
 401         MLX5_EXPANSION_VXLAN,
 402         MLX5_EXPANSION_VXLAN_GPE,
 403         MLX5_EXPANSION_GRE,
 404         MLX5_EXPANSION_MPLS,
 405         MLX5_EXPANSION_ETH,
 406         MLX5_EXPANSION_ETH_VLAN,
 407         MLX5_EXPANSION_VLAN,
 408         MLX5_EXPANSION_IPV4,
 409         MLX5_EXPANSION_IPV4_UDP,
 410         MLX5_EXPANSION_IPV4_TCP,
 411         MLX5_EXPANSION_IPV6,
 412         MLX5_EXPANSION_IPV6_UDP,
 413         MLX5_EXPANSION_IPV6_TCP,
 414 };
 415
 416 /** Supported expansion of items. */
 417 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 418         [MLX5_EXPANSION_ROOT] = {
 419                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 420                                                   MLX5_EXPANSION_IPV4,
 421                                                   MLX5_EXPANSION_IPV6),
 422                 .type = RTE_FLOW_ITEM_TYPE_END,
 423         },
 424         [MLX5_EXPANSION_ROOT_OUTER] = {
 425                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 426                                                   MLX5_EXPANSION_OUTER_IPV4,
 427                                                   MLX5_EXPANSION_OUTER_IPV6),
 428                 .type = RTE_FLOW_ITEM_TYPE_END,
 429         },
 430         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 431                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 432                 .type = RTE_FLOW_ITEM_TYPE_END,
 433         },
 434         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 435                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 436                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 437                 .type = RTE_FLOW_ITEM_TYPE_END,
 438         },
 439         [MLX5_EXPANSION_OUTER_ETH] = {
 440                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 441                                                   MLX5_EXPANSION_OUTER_IPV6,
 442                                                   MLX5_EXPANSION_MPLS),
 443                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 444                 .rss_types = 0,
 445         },
 446         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 447                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 448                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 449                 .rss_types = 0,
 450         },
 451         [MLX5_EXPANSION_OUTER_VLAN] = {
 452                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 453                                                   MLX5_EXPANSION_OUTER_IPV6),
 454                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 455         },
 456         [MLX5_EXPANSION_OUTER_IPV4] = {
 457                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 458                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 459                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 460                          MLX5_EXPANSION_GRE,
 461                          MLX5_EXPANSION_IPV4,
 462                          MLX5_EXPANSION_IPV6),
 463                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 464                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 465                         ETH_RSS_NONFRAG_IPV4_OTHER,
 466         },
 467         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 468                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 469                                                   MLX5_EXPANSION_VXLAN_GPE),
 470                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 471                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 472         },
 473         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 474                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 475                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 476         },
 477         [MLX5_EXPANSION_OUTER_IPV6] = {
 478                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 479                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 480                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 481                          MLX5_EXPANSION_IPV4,
 482                          MLX5_EXPANSION_IPV6),
 483                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 484                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 485                         ETH_RSS_NONFRAG_IPV6_OTHER,
 486         },
 487         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 488                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 489                                                   MLX5_EXPANSION_VXLAN_GPE),
 490                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 491                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 492         },
 493         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 494                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 495                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 496         },
 497         [MLX5_EXPANSION_VXLAN] = {
 498                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 499                                                   MLX5_EXPANSION_IPV4,
 500                                                   MLX5_EXPANSION_IPV6),
 501                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 502         },
 503         [MLX5_EXPANSION_VXLAN_GPE] = {
 504                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 505                                                   MLX5_EXPANSION_IPV4,
 506                                                   MLX5_EXPANSION_IPV6),
 507                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 508         },
 509         [MLX5_EXPANSION_GRE] = {
 510                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 511                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 512         },
 513         [MLX5_EXPANSION_MPLS] = {
 514                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 515                                                   MLX5_EXPANSION_IPV6),
 516                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 517         },
 518         [MLX5_EXPANSION_ETH] = {
 519                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 520                                                   MLX5_EXPANSION_IPV6),
 521                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 522         },
 523         [MLX5_EXPANSION_ETH_VLAN] = {
 524                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 525                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 526         },
 527         [MLX5_EXPANSION_VLAN] = {
 528                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 529                                                   MLX5_EXPANSION_IPV6),
 530                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 531         },
 532         [MLX5_EXPANSION_IPV4] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 534                                                   MLX5_EXPANSION_IPV4_TCP),
 535                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 536                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 537                         ETH_RSS_NONFRAG_IPV4_OTHER,
 538         },
 539         [MLX5_EXPANSION_IPV4_UDP] = {
 540                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 541                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 542         },
 543         [MLX5_EXPANSION_IPV4_TCP] = {
 544                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 545                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 546         },
 547         [MLX5_EXPANSION_IPV6] = {
 548                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 549                                                   MLX5_EXPANSION_IPV6_TCP),
 550                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 551                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 552                         ETH_RSS_NONFRAG_IPV6_OTHER,
 553         },
 554         [MLX5_EXPANSION_IPV6_UDP] = {
 555                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 556                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 557         },
 558         [MLX5_EXPANSION_IPV6_TCP] = {
 559                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 560                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 561         },
 562 };
 563
 564 static struct rte_flow_shared_action *
 565 mlx5_shared_action_create(struct rte_eth_dev *dev,
 566                           const struct rte_flow_shared_action_conf *conf,
 567                           const struct rte_flow_action *action,
 568                           struct rte_flow_error *error);
 569 static int mlx5_shared_action_destroy
 570                                 (struct rte_eth_dev *dev,
 571                                  struct rte_flow_shared_action *shared_action,
 572                                  struct rte_flow_error *error);
 573 static int mlx5_shared_action_update
 574                                 (struct rte_eth_dev *dev,
 575                                  struct rte_flow_shared_action *shared_action,
 576                                  const struct rte_flow_action *action,
 577                                  struct rte_flow_error *error);
 578 static int mlx5_shared_action_query
 579                                 (struct rte_eth_dev *dev,
 580                                  const struct rte_flow_shared_action *action,
 581                                  void *data,
 582                                  struct rte_flow_error *error);
 583 static inline bool
 584 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
 585                           struct rte_flow_tunnel *tunnel,
 586                           const char *err_msg)
 587 {
 588         err_msg = NULL;
 589         if (!is_tunnel_offload_active(dev)) {
 590                 err_msg = "tunnel offload was not activated";
 591                 goto out;
 592         } else if (!tunnel) {
 593                 err_msg = "no application tunnel";
 594                 goto out;
 595         }
 596
 597         switch (tunnel->type) {
 598         default:
 599                 err_msg = "unsupported tunnel type";
 600                 goto out;
 601         case RTE_FLOW_ITEM_TYPE_VXLAN:
 602                 break;
 603         }
 604
 605 out:
 606         return !err_msg;
 607 }
 608
 609
 610 static int
 611 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 612                     struct rte_flow_tunnel *app_tunnel,
 613                     struct rte_flow_action **actions,
 614                     uint32_t *num_of_actions,
 615                     struct rte_flow_error *error)
 616 {
 617         int ret;
 618         struct mlx5_flow_tunnel *tunnel;
 619         const char *err_msg = NULL;
 620         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
 621
 622         if (!verdict)
 623                 return rte_flow_error_set(error, EINVAL,
 624                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
 625                                           err_msg);
 626         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
 627         if (ret < 0) {
 628                 return rte_flow_error_set(error, ret,
 629                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
 630                                           "failed to initialize pmd tunnel");
 631         }
 632         *actions = &tunnel->action;
 633         *num_of_actions = 1;
 634         return 0;
 635 }
 636
 637 static int
 638 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 639                        struct rte_flow_tunnel *app_tunnel,
 640                        struct rte_flow_item **items,
 641                        uint32_t *num_of_items,
 642                        struct rte_flow_error *error)
 643 {
 644         int ret;
 645         struct mlx5_flow_tunnel *tunnel;
 646         const char *err_msg = NULL;
 647         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
 648
 649         if (!verdict)
 650                 return rte_flow_error_set(error, EINVAL,
 651                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 652                                           err_msg);
 653         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
 654         if (ret < 0) {
 655                 return rte_flow_error_set(error, ret,
 656                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 657                                           "failed to initialize pmd tunnel");
 658         }
 659         *items = &tunnel->item;
 660         *num_of_items = 1;
 661         return 0;
 662 }
 663
 664 static int
 665 mlx5_flow_item_release(struct rte_eth_dev *dev,
 666                        struct rte_flow_item *pmd_items,
 667                        uint32_t num_items, struct rte_flow_error *err)
 668 {
 669         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 670         struct mlx5_flow_tunnel *tun;
 671
 672         LIST_FOREACH(tun, &thub->tunnels, chain) {
 673                 if (&tun->item == pmd_items)
 674                         break;
 675         }
 676         if (!tun || num_items != 1)
 677                 return rte_flow_error_set(err, EINVAL,
 678                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 679                                           "invalid argument");
 680         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
 681                 mlx5_flow_tunnel_free(dev, tun);
 682         return 0;
 683 }
 684
 685 static int
 686 mlx5_flow_action_release(struct rte_eth_dev *dev,
 687                          struct rte_flow_action *pmd_actions,
 688                          uint32_t num_actions, struct rte_flow_error *err)
 689 {
 690         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 691         struct mlx5_flow_tunnel *tun;
 692
 693         LIST_FOREACH(tun, &thub->tunnels, chain) {
 694                 if (&tun->action == pmd_actions)
 695                         break;
 696         }
 697         if (!tun || num_actions != 1)
 698                 return rte_flow_error_set(err, EINVAL,
 699                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 700                                           "invalid argument");
 701         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
 702                 mlx5_flow_tunnel_free(dev, tun);
 703
 704         return 0;
 705 }
 706
 707 static int
 708 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 709                                   struct rte_mbuf *m,
 710                                   struct rte_flow_restore_info *info,
 711                                   struct rte_flow_error *err)
 712 {
 713         uint64_t ol_flags = m->ol_flags;
 714         const struct mlx5_flow_tbl_data_entry *tble;
 715         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
 716
 717         if ((ol_flags & mask) != mask)
 718                 goto err;
 719         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
 720         if (!tble) {
 721                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
 722                         dev->data->port_id, m->hash.fdir.hi);
 723                 goto err;
 724         }
 725         MLX5_ASSERT(tble->tunnel);
 726         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
 727         info->group_id = tble->group_id;
 728         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
 729                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
 730                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
 731
 732         return 0;
 733
 734 err:
 735         return rte_flow_error_set(err, EINVAL,
 736                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 737                                   "failed to get restore info");
 738 }
 739
 740 static const struct rte_flow_ops mlx5_flow_ops = {
 741         .validate = mlx5_flow_validate,
 742         .create = mlx5_flow_create,
 743         .destroy = mlx5_flow_destroy,
 744         .flush = mlx5_flow_flush,
 745         .isolate = mlx5_flow_isolate,
 746         .query = mlx5_flow_query,
 747         .dev_dump = mlx5_flow_dev_dump,
 748         .get_aged_flows = mlx5_flow_get_aged_flows,
 749         .shared_action_create = mlx5_shared_action_create,
 750         .shared_action_destroy = mlx5_shared_action_destroy,
 751         .shared_action_update = mlx5_shared_action_update,
 752         .shared_action_query = mlx5_shared_action_query,
 753         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 754         .tunnel_match = mlx5_flow_tunnel_match,
 755         .tunnel_action_decap_release = mlx5_flow_action_release,
 756         .tunnel_item_release = mlx5_flow_item_release,
 757         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 758 };
 759
 760 /* Convert FDIR request to Generic flow. */
 761 struct mlx5_fdir {
 762         struct rte_flow_attr attr;
 763         struct rte_flow_item items[4];
 764         struct rte_flow_item_eth l2;
 765         struct rte_flow_item_eth l2_mask;
 766         union {
 767                 struct rte_flow_item_ipv4 ipv4;
 768                 struct rte_flow_item_ipv6 ipv6;
 769         } l3;
 770         union {
 771                 struct rte_flow_item_ipv4 ipv4;
 772                 struct rte_flow_item_ipv6 ipv6;
 773         } l3_mask;
 774         union {
 775                 struct rte_flow_item_udp udp;
 776                 struct rte_flow_item_tcp tcp;
 777         } l4;
 778         union {
 779                 struct rte_flow_item_udp udp;
 780                 struct rte_flow_item_tcp tcp;
 781         } l4_mask;
 782         struct rte_flow_action actions[2];
 783         struct rte_flow_action_queue queue;
 784 };
 785
 786 /* Tunnel information. */
 787 struct mlx5_flow_tunnel_info {
 788         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 789         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 790 };
 791
 792 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 793         {
 794                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 795                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 796         },
 797         {
 798                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 799                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 800         },
 801         {
 802                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 803                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 804         },
 805         {
 806                 .tunnel = MLX5_FLOW_LAYER_GRE,
 807                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 808         },
 809         {
 810                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 811                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 812         },
 813         {
 814                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 815                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 816         },
 817         {
 818                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 819                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 820         },
 821         {
 822                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 823                 .ptype = RTE_PTYPE_TUNNEL_IP,
 824         },
 825         {
 826                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 827                 .ptype = RTE_PTYPE_TUNNEL_IP,
 828         },
 829         {
 830                 .tunnel = MLX5_FLOW_LAYER_GTP,
 831                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 832         },
 833 };
 834
 835 /**
 836  * Translate tag ID to register.
 837  *
 838  * @param[in] dev
 839  *   Pointer to the Ethernet device structure.
 840  * @param[in] feature
 841  *   The feature that request the register.
 842  * @param[in] id
 843  *   The request register ID.
 844  * @param[out] error
 845  *   Error description in case of any.
 846  *
 847  * @return
 848  *   The request register on success, a negative errno
 849  *   value otherwise and rte_errno is set.
 850  */
 851 int
 852 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 853                      enum mlx5_feature_name feature,
 854                      uint32_t id,
 855                      struct rte_flow_error *error)
 856 {
 857         struct mlx5_priv *priv = dev->data->dev_private;
 858         struct mlx5_dev_config *config = &priv->config;
 859         enum modify_reg start_reg;
 860         bool skip_mtr_reg = false;
 861
 862         switch (feature) {
 863         case MLX5_HAIRPIN_RX:
 864                 return REG_B;
 865         case MLX5_HAIRPIN_TX:
 866                 return REG_A;
 867         case MLX5_METADATA_RX:
 868                 switch (config->dv_xmeta_en) {
 869                 case MLX5_XMETA_MODE_LEGACY:
 870                         return REG_B;
 871                 case MLX5_XMETA_MODE_META16:
 872                         return REG_C_0;
 873                 case MLX5_XMETA_MODE_META32:
 874                         return REG_C_1;
 875                 }
 876                 break;
 877         case MLX5_METADATA_TX:
 878                 return REG_A;
 879         case MLX5_METADATA_FDB:
 880                 switch (config->dv_xmeta_en) {
 881                 case MLX5_XMETA_MODE_LEGACY:
 882                         return REG_NON;
 883                 case MLX5_XMETA_MODE_META16:
 884                         return REG_C_0;
 885                 case MLX5_XMETA_MODE_META32:
 886                         return REG_C_1;
 887                 }
 888                 break;
 889         case MLX5_FLOW_MARK:
 890                 switch (config->dv_xmeta_en) {
 891                 case MLX5_XMETA_MODE_LEGACY:
 892                         return REG_NON;
 893                 case MLX5_XMETA_MODE_META16:
 894                         return REG_C_1;
 895                 case MLX5_XMETA_MODE_META32:
 896                         return REG_C_0;
 897                 }
 898                 break;
 899         case MLX5_MTR_SFX:
 900                 /*
 901                  * If meter color and flow match share one register, flow match
 902                  * should use the meter color register for match.
 903                  */
 904                 if (priv->mtr_reg_share)
 905                         return priv->mtr_color_reg;
 906                 else
 907                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 908                                REG_C_3;
 909         case MLX5_MTR_COLOR:
 910                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 911                 return priv->mtr_color_reg;
 912         case MLX5_COPY_MARK:
 913                 /*
 914                  * Metadata COPY_MARK register using is in meter suffix sub
 915                  * flow while with meter. It's safe to share the same register.
 916                  */
 917                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 918         case MLX5_APP_TAG:
 919                 /*
 920                  * If meter is enable, it will engage the register for color
 921                  * match and flow match. If meter color match is not using the
 922                  * REG_C_2, need to skip the REG_C_x be used by meter color
 923                  * match.
 924                  * If meter is disable, free to use all available registers.
 925                  */
 926                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 927                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 928                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 929                 if (id > (REG_C_7 - start_reg))
 930                         return rte_flow_error_set(error, EINVAL,
 931                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 932                                                   NULL, "invalid tag id");
 933                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 934                         return rte_flow_error_set(error, ENOTSUP,
 935                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 936                                                   NULL, "unsupported tag id");
 937                 /*
 938                  * This case means meter is using the REG_C_x great than 2.
 939                  * Take care not to conflict with meter color REG_C_x.
 940                  * If the available index REG_C_y >= REG_C_x, skip the
 941                  * color register.
 942                  */
 943                 if (skip_mtr_reg && config->flow_mreg_c
 944                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 945                         if (id >= (REG_C_7 - start_reg))
 946                                 return rte_flow_error_set(error, EINVAL,
 947                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 948                                                         NULL, "invalid tag id");
 949                         if (config->flow_mreg_c
 950                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 951                                 return config->flow_mreg_c
 952                                                [id + 1 + start_reg - REG_C_0];
 953                         return rte_flow_error_set(error, ENOTSUP,
 954                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 955                                                   NULL, "unsupported tag id");
 956                 }
 957                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 958         }
 959         MLX5_ASSERT(false);
 960         return rte_flow_error_set(error, EINVAL,
 961                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 962                                   NULL, "invalid feature name");
 963 }
 964
 965 /**
 966  * Check extensive flow metadata register support.
 967  *
 968  * @param dev
 969  *   Pointer to rte_eth_dev structure.
 970  *
 971  * @return
 972  *   True if device supports extensive flow metadata register, otherwise false.
 973  */
 974 bool
 975 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 976 {
 977         struct mlx5_priv *priv = dev->data->dev_private;
 978         struct mlx5_dev_config *config = &priv->config;
 979
 980         /*
 981          * Having available reg_c can be regarded inclusively as supporting
 982          * extensive flow metadata register, which could mean,
 983          * - metadata register copy action by modify header.
 984          * - 16 modify header actions is supported.
 985          * - reg_c's are preserved across different domain (FDB and NIC) on
 986          *   packet loopback by flow lookup miss.
 987          */
 988         return config->flow_mreg_c[2] != REG_NON;
 989 }
 990
 991 /**
 992  * Verify the @p item specifications (spec, last, mask) are compatible with the
 993  * NIC capabilities.
 994  *
 995  * @param[in] item
 996  *   Item specification.
 997  * @param[in] mask
 998  *   @p item->mask or flow default bit-masks.
 999  * @param[in] nic_mask
1000  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1001  * @param[in] size
1002  *   Bit-masks size in bytes.
1003  * @param[in] range_accepted
1004  *   True if range of values is accepted for specific fields, false otherwise.
1005  * @param[out] error
1006  *   Pointer to error structure.
1007  *
1008  * @return
1009  *   0 on success, a negative errno value otherwise and rte_errno is set.
1010  */
1011 int
1012 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1013                           const uint8_t *mask,
1014                           const uint8_t *nic_mask,
1015                           unsigned int size,
1016                           bool range_accepted,
1017                           struct rte_flow_error *error)
1018 {
1019         unsigned int i;
1020
1021         MLX5_ASSERT(nic_mask);
1022         for (i = 0; i < size; ++i)
1023                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1024                         return rte_flow_error_set(error, ENOTSUP,
1025                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1026                                                   item,
1027                                                   "mask enables non supported"
1028                                                   " bits");
1029         if (!item->spec && (item->mask || item->last))
1030                 return rte_flow_error_set(error, EINVAL,
1031                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1032                                           "mask/last without a spec is not"
1033                                           " supported");
1034         if (item->spec && item->last && !range_accepted) {
1035                 uint8_t spec[size];
1036                 uint8_t last[size];
1037                 unsigned int i;
1038                 int ret;
1039
1040                 for (i = 0; i < size; ++i) {
1041                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1042                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1043                 }
1044                 ret = memcmp(spec, last, size);
1045                 if (ret != 0)
1046                         return rte_flow_error_set(error, EINVAL,
1047                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1048                                                   item,
1049                                                   "range is not valid");
1050         }
1051         return 0;
1052 }
1053
1054 /**
1055  * Adjust the hash fields according to the @p flow information.
1056  *
1057  * @param[in] dev_flow.
1058  *   Pointer to the mlx5_flow.
1059  * @param[in] tunnel
1060  *   1 when the hash field is for a tunnel item.
1061  * @param[in] layer_types
1062  *   ETH_RSS_* types.
1063  * @param[in] hash_fields
1064  *   Item hash fields.
1065  *
1066  * @return
1067  *   The hash fields that should be used.
1068  */
1069 uint64_t
1070 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1071                             int tunnel __rte_unused, uint64_t layer_types,
1072                             uint64_t hash_fields)
1073 {
1074 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1075         int rss_request_inner = rss_desc->level >= 2;
1076
1077         /* Check RSS hash level for tunnel. */
1078         if (tunnel && rss_request_inner)
1079                 hash_fields |= IBV_RX_HASH_INNER;
1080         else if (tunnel || rss_request_inner)
1081                 return 0;
1082 #endif
1083         /* Check if requested layer matches RSS hash fields. */
1084         if (!(rss_desc->types & layer_types))
1085                 return 0;
1086         return hash_fields;
1087 }
1088
1089 /**
1090  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1091  * if several tunnel rules are used on this queue, the tunnel ptype will be
1092  * cleared.
1093  *
1094  * @param rxq_ctrl
1095  *   Rx queue to update.
1096  */
1097 static void
1098 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1099 {
1100         unsigned int i;
1101         uint32_t tunnel_ptype = 0;
1102
1103         /* Look up for the ptype to use. */
1104         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1105                 if (!rxq_ctrl->flow_tunnels_n[i])
1106                         continue;
1107                 if (!tunnel_ptype) {
1108                         tunnel_ptype = tunnels_info[i].ptype;
1109                 } else {
1110                         tunnel_ptype = 0;
1111                         break;
1112                 }
1113         }
1114         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1115 }
1116
1117 /**
1118  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1119  * flow.
1120  *
1121  * @param[in] dev
1122  *   Pointer to the Ethernet device structure.
1123  * @param[in] dev_handle
1124  *   Pointer to device flow handle structure.
1125  */
1126 static void
1127 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1128                        struct mlx5_flow_handle *dev_handle)
1129 {
1130         struct mlx5_priv *priv = dev->data->dev_private;
1131         const int mark = dev_handle->mark;
1132         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1133         struct mlx5_hrxq *hrxq;
1134         unsigned int i;
1135
1136         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1137                 return;
1138         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1139                               dev_handle->rix_hrxq);
1140         if (!hrxq)
1141                 return;
1142         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1143                 int idx = hrxq->ind_table->queues[i];
1144                 struct mlx5_rxq_ctrl *rxq_ctrl =
1145                         container_of((*priv->rxqs)[idx],
1146                                      struct mlx5_rxq_ctrl, rxq);
1147
1148                 /*
1149                  * To support metadata register copy on Tx loopback,
1150                  * this must be always enabled (metadata may arive
1151                  * from other port - not from local flows only.
1152                  */
1153                 if (priv->config.dv_flow_en &&
1154                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1155                     mlx5_flow_ext_mreg_supported(dev)) {
1156                         rxq_ctrl->rxq.mark = 1;
1157                         rxq_ctrl->flow_mark_n = 1;
1158                 } else if (mark) {
1159                         rxq_ctrl->rxq.mark = 1;
1160                         rxq_ctrl->flow_mark_n++;
1161                 }
1162                 if (tunnel) {
1163                         unsigned int j;
1164
1165                         /* Increase the counter matching the flow. */
1166                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1167                                 if ((tunnels_info[j].tunnel &
1168                                      dev_handle->layers) ==
1169                                     tunnels_info[j].tunnel) {
1170                                         rxq_ctrl->flow_tunnels_n[j]++;
1171                                         break;
1172                                 }
1173                         }
1174                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1175                 }
1176         }
1177 }
1178
1179 /**
1180  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1181  *
1182  * @param[in] dev
1183  *   Pointer to the Ethernet device structure.
1184  * @param[in] flow
1185  *   Pointer to flow structure.
1186  */
1187 static void
1188 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1189 {
1190         struct mlx5_priv *priv = dev->data->dev_private;
1191         uint32_t handle_idx;
1192         struct mlx5_flow_handle *dev_handle;
1193
1194         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1195                        handle_idx, dev_handle, next)
1196                 flow_drv_rxq_flags_set(dev, dev_handle);
1197 }
1198
1199 /**
1200  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1201  * device flow if no other flow uses it with the same kind of request.
1202  *
1203  * @param dev
1204  *   Pointer to Ethernet device.
1205  * @param[in] dev_handle
1206  *   Pointer to the device flow handle structure.
1207  */
1208 static void
1209 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1210                         struct mlx5_flow_handle *dev_handle)
1211 {
1212         struct mlx5_priv *priv = dev->data->dev_private;
1213         const int mark = dev_handle->mark;
1214         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1215         struct mlx5_hrxq *hrxq;
1216         unsigned int i;
1217
1218         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1219                 return;
1220         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1221                               dev_handle->rix_hrxq);
1222         if (!hrxq)
1223                 return;
1224         MLX5_ASSERT(dev->data->dev_started);
1225         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1226                 int idx = hrxq->ind_table->queues[i];
1227                 struct mlx5_rxq_ctrl *rxq_ctrl =
1228                         container_of((*priv->rxqs)[idx],
1229                                      struct mlx5_rxq_ctrl, rxq);
1230
1231                 if (priv->config.dv_flow_en &&
1232                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1233                     mlx5_flow_ext_mreg_supported(dev)) {
1234                         rxq_ctrl->rxq.mark = 1;
1235                         rxq_ctrl->flow_mark_n = 1;
1236                 } else if (mark) {
1237                         rxq_ctrl->flow_mark_n--;
1238                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1239                 }
1240                 if (tunnel) {
1241                         unsigned int j;
1242
1243                         /* Decrease the counter matching the flow. */
1244                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1245                                 if ((tunnels_info[j].tunnel &
1246                                      dev_handle->layers) ==
1247                                     tunnels_info[j].tunnel) {
1248                                         rxq_ctrl->flow_tunnels_n[j]--;
1249                                         break;
1250                                 }
1251                         }
1252                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1253                 }
1254         }
1255 }
1256
1257 /**
1258  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1259  * @p flow if no other flow uses it with the same kind of request.
1260  *
1261  * @param dev
1262  *   Pointer to Ethernet device.
1263  * @param[in] flow
1264  *   Pointer to the flow.
1265  */
1266 static void
1267 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1268 {
1269         struct mlx5_priv *priv = dev->data->dev_private;
1270         uint32_t handle_idx;
1271         struct mlx5_flow_handle *dev_handle;
1272
1273         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1274                        handle_idx, dev_handle, next)
1275                 flow_drv_rxq_flags_trim(dev, dev_handle);
1276 }
1277
1278 /**
1279  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1280  *
1281  * @param dev
1282  *   Pointer to Ethernet device.
1283  */
1284 static void
1285 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1286 {
1287         struct mlx5_priv *priv = dev->data->dev_private;
1288         unsigned int i;
1289
1290         for (i = 0; i != priv->rxqs_n; ++i) {
1291                 struct mlx5_rxq_ctrl *rxq_ctrl;
1292                 unsigned int j;
1293
1294                 if (!(*priv->rxqs)[i])
1295                         continue;
1296                 rxq_ctrl = container_of((*priv->rxqs)[i],
1297                                         struct mlx5_rxq_ctrl, rxq);
1298                 rxq_ctrl->flow_mark_n = 0;
1299                 rxq_ctrl->rxq.mark = 0;
1300                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1301                         rxq_ctrl->flow_tunnels_n[j] = 0;
1302                 rxq_ctrl->rxq.tunnel = 0;
1303         }
1304 }
1305
1306 /**
1307  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1308  *
1309  * @param[in] dev
1310  *   Pointer to the Ethernet device structure.
1311  */
1312 void
1313 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1314 {
1315         struct mlx5_priv *priv = dev->data->dev_private;
1316         struct mlx5_rxq_data *data;
1317         unsigned int i;
1318
1319         for (i = 0; i != priv->rxqs_n; ++i) {
1320                 if (!(*priv->rxqs)[i])
1321                         continue;
1322                 data = (*priv->rxqs)[i];
1323                 if (!rte_flow_dynf_metadata_avail()) {
1324                         data->dynf_meta = 0;
1325                         data->flow_meta_mask = 0;
1326                         data->flow_meta_offset = -1;
1327                 } else {
1328                         data->dynf_meta = 1;
1329                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1330                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1331                 }
1332         }
1333 }
1334
1335 /*
1336  * return a pointer to the desired action in the list of actions.
1337  *
1338  * @param[in] actions
1339  *   The list of actions to search the action in.
1340  * @param[in] action
1341  *   The action to find.
1342  *
1343  * @return
1344  *   Pointer to the action in the list, if found. NULL otherwise.
1345  */
1346 const struct rte_flow_action *
1347 mlx5_flow_find_action(const struct rte_flow_action *actions,
1348                       enum rte_flow_action_type action)
1349 {
1350         if (actions == NULL)
1351                 return NULL;
1352         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1353                 if (actions->type == action)
1354                         return actions;
1355         return NULL;
1356 }
1357
1358 /*
1359  * Validate the flag action.
1360  *
1361  * @param[in] action_flags
1362  *   Bit-fields that holds the actions detected until now.
1363  * @param[in] attr
1364  *   Attributes of flow that includes this action.
1365  * @param[out] error
1366  *   Pointer to error structure.
1367  *
1368  * @return
1369  *   0 on success, a negative errno value otherwise and rte_errno is set.
1370  */
1371 int
1372 mlx5_flow_validate_action_flag(uint64_t action_flags,
1373                                const struct rte_flow_attr *attr,
1374                                struct rte_flow_error *error)
1375 {
1376         if (action_flags & MLX5_FLOW_ACTION_MARK)
1377                 return rte_flow_error_set(error, EINVAL,
1378                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1379                                           "can't mark and flag in same flow");
1380         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1381                 return rte_flow_error_set(error, EINVAL,
1382                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1383                                           "can't have 2 flag"
1384                                           " actions in same flow");
1385         if (attr->egress)
1386                 return rte_flow_error_set(error, ENOTSUP,
1387                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1388                                           "flag action not supported for "
1389                                           "egress");
1390         return 0;
1391 }
1392
1393 /*
1394  * Validate the mark action.
1395  *
1396  * @param[in] action
1397  *   Pointer to the queue action.
1398  * @param[in] action_flags
1399  *   Bit-fields that holds the actions detected until now.
1400  * @param[in] attr
1401  *   Attributes of flow that includes this action.
1402  * @param[out] error
1403  *   Pointer to error structure.
1404  *
1405  * @return
1406  *   0 on success, a negative errno value otherwise and rte_errno is set.
1407  */
1408 int
1409 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1410                                uint64_t action_flags,
1411                                const struct rte_flow_attr *attr,
1412                                struct rte_flow_error *error)
1413 {
1414         const struct rte_flow_action_mark *mark = action->conf;
1415
1416         if (!mark)
1417                 return rte_flow_error_set(error, EINVAL,
1418                                           RTE_FLOW_ERROR_TYPE_ACTION,
1419                                           action,
1420                                           "configuration cannot be null");
1421         if (mark->id >= MLX5_FLOW_MARK_MAX)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1424                                           &mark->id,
1425                                           "mark id must in 0 <= id < "
1426                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1427         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1428                 return rte_flow_error_set(error, EINVAL,
1429                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1430                                           "can't flag and mark in same flow");
1431         if (action_flags & MLX5_FLOW_ACTION_MARK)
1432                 return rte_flow_error_set(error, EINVAL,
1433                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1434                                           "can't have 2 mark actions in same"
1435                                           " flow");
1436         if (attr->egress)
1437                 return rte_flow_error_set(error, ENOTSUP,
1438                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1439                                           "mark action not supported for "
1440                                           "egress");
1441         return 0;
1442 }
1443
1444 /*
1445  * Validate the drop action.
1446  *
1447  * @param[in] action_flags
1448  *   Bit-fields that holds the actions detected until now.
1449  * @param[in] attr
1450  *   Attributes of flow that includes this action.
1451  * @param[out] error
1452  *   Pointer to error structure.
1453  *
1454  * @return
1455  *   0 on success, a negative errno value otherwise and rte_errno is set.
1456  */
1457 int
1458 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1459                                const struct rte_flow_attr *attr,
1460                                struct rte_flow_error *error)
1461 {
1462         if (attr->egress)
1463                 return rte_flow_error_set(error, ENOTSUP,
1464                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1465                                           "drop action not supported for "
1466                                           "egress");
1467         return 0;
1468 }
1469
1470 /*
1471  * Validate the queue action.
1472  *
1473  * @param[in] action
1474  *   Pointer to the queue action.
1475  * @param[in] action_flags
1476  *   Bit-fields that holds the actions detected until now.
1477  * @param[in] dev
1478  *   Pointer to the Ethernet device structure.
1479  * @param[in] attr
1480  *   Attributes of flow that includes this action.
1481  * @param[out] error
1482  *   Pointer to error structure.
1483  *
1484  * @return
1485  *   0 on success, a negative errno value otherwise and rte_errno is set.
1486  */
1487 int
1488 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1489                                 uint64_t action_flags,
1490                                 struct rte_eth_dev *dev,
1491                                 const struct rte_flow_attr *attr,
1492                                 struct rte_flow_error *error)
1493 {
1494         struct mlx5_priv *priv = dev->data->dev_private;
1495         const struct rte_flow_action_queue *queue = action->conf;
1496
1497         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1500                                           "can't have 2 fate actions in"
1501                                           " same flow");
1502         if (!priv->rxqs_n)
1503                 return rte_flow_error_set(error, EINVAL,
1504                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1505                                           NULL, "No Rx queues configured");
1506         if (queue->index >= priv->rxqs_n)
1507                 return rte_flow_error_set(error, EINVAL,
1508                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1509                                           &queue->index,
1510                                           "queue index out of range");
1511         if (!(*priv->rxqs)[queue->index])
1512                 return rte_flow_error_set(error, EINVAL,
1513                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1514                                           &queue->index,
1515                                           "queue is not configured");
1516         if (attr->egress)
1517                 return rte_flow_error_set(error, ENOTSUP,
1518                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1519                                           "queue action not supported for "
1520                                           "egress");
1521         return 0;
1522 }
1523
1524 /*
1525  * Validate the rss action.
1526  *
1527  * @param[in] dev
1528  *   Pointer to the Ethernet device structure.
1529  * @param[in] action
1530  *   Pointer to the queue action.
1531  * @param[out] error
1532  *   Pointer to error structure.
1533  *
1534  * @return
1535  *   0 on success, a negative errno value otherwise and rte_errno is set.
1536  */
1537 int
1538 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1539                          const struct rte_flow_action *action,
1540                          struct rte_flow_error *error)
1541 {
1542         struct mlx5_priv *priv = dev->data->dev_private;
1543         const struct rte_flow_action_rss *rss = action->conf;
1544         unsigned int i;
1545
1546         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1547             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1548                 return rte_flow_error_set(error, ENOTSUP,
1549                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1550                                           &rss->func,
1551                                           "RSS hash function not supported");
1552 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1553         if (rss->level > 2)
1554 #else
1555         if (rss->level > 1)
1556 #endif
1557                 return rte_flow_error_set(error, ENOTSUP,
1558                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1559                                           &rss->level,
1560                                           "tunnel RSS is not supported");
1561         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1562         if (rss->key_len == 0 && rss->key != NULL)
1563                 return rte_flow_error_set(error, ENOTSUP,
1564                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1565                                           &rss->key_len,
1566                                           "RSS hash key length 0");
1567         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1568                 return rte_flow_error_set(error, ENOTSUP,
1569                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1570                                           &rss->key_len,
1571                                           "RSS hash key too small");
1572         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1573                 return rte_flow_error_set(error, ENOTSUP,
1574                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1575                                           &rss->key_len,
1576                                           "RSS hash key too large");
1577         if (rss->queue_num > priv->config.ind_table_max_size)
1578                 return rte_flow_error_set(error, ENOTSUP,
1579                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1580                                           &rss->queue_num,
1581                                           "number of queues too large");
1582         if (rss->types & MLX5_RSS_HF_MASK)
1583                 return rte_flow_error_set(error, ENOTSUP,
1584                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1585                                           &rss->types,
1586                                           "some RSS protocols are not"
1587                                           " supported");
1588         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1589             !(rss->types & ETH_RSS_IP))
1590                 return rte_flow_error_set(error, EINVAL,
1591                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1592                                           "L3 partial RSS requested but L3 RSS"
1593                                           " type not specified");
1594         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1595             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1596                 return rte_flow_error_set(error, EINVAL,
1597                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1598                                           "L4 partial RSS requested but L4 RSS"
1599                                           " type not specified");
1600         if (!priv->rxqs_n)
1601                 return rte_flow_error_set(error, EINVAL,
1602                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1603                                           NULL, "No Rx queues configured");
1604         if (!rss->queue_num)
1605                 return rte_flow_error_set(error, EINVAL,
1606                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1607                                           NULL, "No queues configured");
1608         for (i = 0; i != rss->queue_num; ++i) {
1609                 if (rss->queue[i] >= priv->rxqs_n)
1610                         return rte_flow_error_set
1611                                 (error, EINVAL,
1612                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1613                                  &rss->queue[i], "queue index out of range");
1614                 if (!(*priv->rxqs)[rss->queue[i]])
1615                         return rte_flow_error_set
1616                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1617                                  &rss->queue[i], "queue is not configured");
1618         }
1619         return 0;
1620 }
1621
1622 /*
1623  * Validate the rss action.
1624  *
1625  * @param[in] action
1626  *   Pointer to the queue action.
1627  * @param[in] action_flags
1628  *   Bit-fields that holds the actions detected until now.
1629  * @param[in] dev
1630  *   Pointer to the Ethernet device structure.
1631  * @param[in] attr
1632  *   Attributes of flow that includes this action.
1633  * @param[in] item_flags
1634  *   Items that were detected.
1635  * @param[out] error
1636  *   Pointer to error structure.
1637  *
1638  * @return
1639  *   0 on success, a negative errno value otherwise and rte_errno is set.
1640  */
1641 int
1642 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1643                               uint64_t action_flags,
1644                               struct rte_eth_dev *dev,
1645                               const struct rte_flow_attr *attr,
1646                               uint64_t item_flags,
1647                               struct rte_flow_error *error)
1648 {
1649         const struct rte_flow_action_rss *rss = action->conf;
1650         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1651         int ret;
1652
1653         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1654                 return rte_flow_error_set(error, EINVAL,
1655                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1656                                           "can't have 2 fate actions"
1657                                           " in same flow");
1658         ret = mlx5_validate_action_rss(dev, action, error);
1659         if (ret)
1660                 return ret;
1661         if (attr->egress)
1662                 return rte_flow_error_set(error, ENOTSUP,
1663                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1664                                           "rss action not supported for "
1665                                           "egress");
1666         if (rss->level > 1 && !tunnel)
1667                 return rte_flow_error_set(error, EINVAL,
1668                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1669                                           "inner RSS is not supported for "
1670                                           "non-tunnel flows");
1671         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1672             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1673                 return rte_flow_error_set(error, EINVAL,
1674                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1675                                           "RSS on eCPRI is not supported now");
1676         }
1677         return 0;
1678 }
1679
1680 /*
1681  * Validate the default miss action.
1682  *
1683  * @param[in] action_flags
1684  *   Bit-fields that holds the actions detected until now.
1685  * @param[out] error
1686  *   Pointer to error structure.
1687  *
1688  * @return
1689  *   0 on success, a negative errno value otherwise and rte_errno is set.
1690  */
1691 int
1692 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1693                                 const struct rte_flow_attr *attr,
1694                                 struct rte_flow_error *error)
1695 {
1696         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1697                 return rte_flow_error_set(error, EINVAL,
1698                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1699                                           "can't have 2 fate actions in"
1700                                           " same flow");
1701         if (attr->egress)
1702                 return rte_flow_error_set(error, ENOTSUP,
1703                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1704                                           "default miss action not supported "
1705                                           "for egress");
1706         if (attr->group)
1707                 return rte_flow_error_set(error, ENOTSUP,
1708                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1709                                           "only group 0 is supported");
1710         if (attr->transfer)
1711                 return rte_flow_error_set(error, ENOTSUP,
1712                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1713                                           NULL, "transfer is not supported");
1714         return 0;
1715 }
1716
1717 /*
1718  * Validate the count action.
1719  *
1720  * @param[in] dev
1721  *   Pointer to the Ethernet device structure.
1722  * @param[in] attr
1723  *   Attributes of flow that includes this action.
1724  * @param[out] error
1725  *   Pointer to error structure.
1726  *
1727  * @return
1728  *   0 on success, a negative errno value otherwise and rte_errno is set.
1729  */
1730 int
1731 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1732                                 const struct rte_flow_attr *attr,
1733                                 struct rte_flow_error *error)
1734 {
1735         if (attr->egress)
1736                 return rte_flow_error_set(error, ENOTSUP,
1737                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1738                                           "count action not supported for "
1739                                           "egress");
1740         return 0;
1741 }
1742
1743 /**
1744  * Verify the @p attributes will be correctly understood by the NIC and store
1745  * them in the @p flow if everything is correct.
1746  *
1747  * @param[in] dev
1748  *   Pointer to the Ethernet device structure.
1749  * @param[in] attributes
1750  *   Pointer to flow attributes
1751  * @param[out] error
1752  *   Pointer to error structure.
1753  *
1754  * @return
1755  *   0 on success, a negative errno value otherwise and rte_errno is set.
1756  */
1757 int
1758 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1759                               const struct rte_flow_attr *attributes,
1760                               struct rte_flow_error *error)
1761 {
1762         struct mlx5_priv *priv = dev->data->dev_private;
1763         uint32_t priority_max = priv->config.flow_prio - 1;
1764
1765         if (attributes->group)
1766                 return rte_flow_error_set(error, ENOTSUP,
1767                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1768                                           NULL, "groups is not supported");
1769         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1770             attributes->priority >= priority_max)
1771                 return rte_flow_error_set(error, ENOTSUP,
1772                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1773                                           NULL, "priority out of range");
1774         if (attributes->egress)
1775                 return rte_flow_error_set(error, ENOTSUP,
1776                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1777                                           "egress is not supported");
1778         if (attributes->transfer && !priv->config.dv_esw_en)
1779                 return rte_flow_error_set(error, ENOTSUP,
1780                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1781                                           NULL, "transfer is not supported");
1782         if (!attributes->ingress)
1783                 return rte_flow_error_set(error, EINVAL,
1784                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1785                                           NULL,
1786                                           "ingress attribute is mandatory");
1787         return 0;
1788 }
1789
1790 /**
1791  * Validate ICMP6 item.
1792  *
1793  * @param[in] item
1794  *   Item specification.
1795  * @param[in] item_flags
1796  *   Bit-fields that holds the items detected until now.
1797  * @param[in] ext_vlan_sup
1798  *   Whether extended VLAN features are supported or not.
1799  * @param[out] error
1800  *   Pointer to error structure.
1801  *
1802  * @return
1803  *   0 on success, a negative errno value otherwise and rte_errno is set.
1804  */
1805 int
1806 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1807                                uint64_t item_flags,
1808                                uint8_t target_protocol,
1809                                struct rte_flow_error *error)
1810 {
1811         const struct rte_flow_item_icmp6 *mask = item->mask;
1812         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1813         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1814                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1815         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1816                                       MLX5_FLOW_LAYER_OUTER_L4;
1817         int ret;
1818
1819         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1820                 return rte_flow_error_set(error, EINVAL,
1821                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1822                                           "protocol filtering not compatible"
1823                                           " with ICMP6 layer");
1824         if (!(item_flags & l3m))
1825                 return rte_flow_error_set(error, EINVAL,
1826                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1827                                           "IPv6 is mandatory to filter on"
1828                                           " ICMP6");
1829         if (item_flags & l4m)
1830                 return rte_flow_error_set(error, EINVAL,
1831                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1832                                           "multiple L4 layers not supported");
1833         if (!mask)
1834                 mask = &rte_flow_item_icmp6_mask;
1835         ret = mlx5_flow_item_acceptable
1836                 (item, (const uint8_t *)mask,
1837                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1838                  sizeof(struct rte_flow_item_icmp6),
1839                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1840         if (ret < 0)
1841                 return ret;
1842         return 0;
1843 }
1844
1845 /**
1846  * Validate ICMP item.
1847  *
1848  * @param[in] item
1849  *   Item specification.
1850  * @param[in] item_flags
1851  *   Bit-fields that holds the items detected until now.
1852  * @param[out] error
1853  *   Pointer to error structure.
1854  *
1855  * @return
1856  *   0 on success, a negative errno value otherwise and rte_errno is set.
1857  */
1858 int
1859 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1860                              uint64_t item_flags,
1861                              uint8_t target_protocol,
1862                              struct rte_flow_error *error)
1863 {
1864         const struct rte_flow_item_icmp *mask = item->mask;
1865         const struct rte_flow_item_icmp nic_mask = {
1866                 .hdr.icmp_type = 0xff,
1867                 .hdr.icmp_code = 0xff,
1868                 .hdr.icmp_ident = RTE_BE16(0xffff),
1869                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1870         };
1871         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1872         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1873                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1874         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1875                                       MLX5_FLOW_LAYER_OUTER_L4;
1876         int ret;
1877
1878         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1879                 return rte_flow_error_set(error, EINVAL,
1880                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1881                                           "protocol filtering not compatible"
1882                                           " with ICMP layer");
1883         if (!(item_flags & l3m))
1884                 return rte_flow_error_set(error, EINVAL,
1885                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1886                                           "IPv4 is mandatory to filter"
1887                                           " on ICMP");
1888         if (item_flags & l4m)
1889                 return rte_flow_error_set(error, EINVAL,
1890                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1891                                           "multiple L4 layers not supported");
1892         if (!mask)
1893                 mask = &nic_mask;
1894         ret = mlx5_flow_item_acceptable
1895                 (item, (const uint8_t *)mask,
1896                  (const uint8_t *)&nic_mask,
1897                  sizeof(struct rte_flow_item_icmp),
1898                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1899         if (ret < 0)
1900                 return ret;
1901         return 0;
1902 }
1903
1904 /**
1905  * Validate Ethernet item.
1906  *
1907  * @param[in] item
1908  *   Item specification.
1909  * @param[in] item_flags
1910  *   Bit-fields that holds the items detected until now.
1911  * @param[out] error
1912  *   Pointer to error structure.
1913  *
1914  * @return
1915  *   0 on success, a negative errno value otherwise and rte_errno is set.
1916  */
1917 int
1918 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1919                             uint64_t item_flags, bool ext_vlan_sup,
1920                             struct rte_flow_error *error)
1921 {
1922         const struct rte_flow_item_eth *mask = item->mask;
1923         const struct rte_flow_item_eth nic_mask = {
1924                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1925                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1926                 .type = RTE_BE16(0xffff),
1927                 .has_vlan = ext_vlan_sup ? 1 : 0,
1928         };
1929         int ret;
1930         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1931         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1932                                        MLX5_FLOW_LAYER_OUTER_L2;
1933
1934         if (item_flags & ethm)
1935                 return rte_flow_error_set(error, ENOTSUP,
1936                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1937                                           "multiple L2 layers not supported");
1938         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1939             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1940                 return rte_flow_error_set(error, EINVAL,
1941                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1942                                           "L2 layer should not follow "
1943                                           "L3 layers");
1944         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1945             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "L2 layer should not follow VLAN");
1949         if (!mask)
1950                 mask = &rte_flow_item_eth_mask;
1951         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1952                                         (const uint8_t *)&nic_mask,
1953                                         sizeof(struct rte_flow_item_eth),
1954                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1955         return ret;
1956 }
1957
1958 /**
1959  * Validate VLAN item.
1960  *
1961  * @param[in] item
1962  *   Item specification.
1963  * @param[in] item_flags
1964  *   Bit-fields that holds the items detected until now.
1965  * @param[in] dev
1966  *   Ethernet device flow is being created on.
1967  * @param[out] error
1968  *   Pointer to error structure.
1969  *
1970  * @return
1971  *   0 on success, a negative errno value otherwise and rte_errno is set.
1972  */
1973 int
1974 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1975                              uint64_t item_flags,
1976                              struct rte_eth_dev *dev,
1977                              struct rte_flow_error *error)
1978 {
1979         const struct rte_flow_item_vlan *spec = item->spec;
1980         const struct rte_flow_item_vlan *mask = item->mask;
1981         const struct rte_flow_item_vlan nic_mask = {
1982                 .tci = RTE_BE16(UINT16_MAX),
1983                 .inner_type = RTE_BE16(UINT16_MAX),
1984         };
1985         uint16_t vlan_tag = 0;
1986         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1987         int ret;
1988         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1989                                         MLX5_FLOW_LAYER_INNER_L4) :
1990                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1991                                         MLX5_FLOW_LAYER_OUTER_L4);
1992         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1993                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1994
1995         if (item_flags & vlanm)
1996                 return rte_flow_error_set(error, EINVAL,
1997                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1998                                           "multiple VLAN layers not supported");
1999         else if ((item_flags & l34m) != 0)
2000                 return rte_flow_error_set(error, EINVAL,
2001                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2002                                           "VLAN cannot follow L3/L4 layer");
2003         if (!mask)
2004                 mask = &rte_flow_item_vlan_mask;
2005         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2006                                         (const uint8_t *)&nic_mask,
2007                                         sizeof(struct rte_flow_item_vlan),
2008                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2009         if (ret)
2010                 return ret;
2011         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2012                 struct mlx5_priv *priv = dev->data->dev_private;
2013
2014                 if (priv->vmwa_context) {
2015                         /*
2016                          * Non-NULL context means we have a virtual machine
2017                          * and SR-IOV enabled, we have to create VLAN interface
2018                          * to make hypervisor to setup E-Switch vport
2019                          * context correctly. We avoid creating the multiple
2020                          * VLAN interfaces, so we cannot support VLAN tag mask.
2021                          */
2022                         return rte_flow_error_set(error, EINVAL,
2023                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2024                                                   item,
2025                                                   "VLAN tag mask is not"
2026                                                   " supported in virtual"
2027                                                   " environment");
2028                 }
2029         }
2030         if (spec) {
2031                 vlan_tag = spec->tci;
2032                 vlan_tag &= mask->tci;
2033         }
2034         /*
2035          * From verbs perspective an empty VLAN is equivalent
2036          * to a packet without VLAN layer.
2037          */
2038         if (!vlan_tag)
2039                 return rte_flow_error_set(error, EINVAL,
2040                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2041                                           item->spec,
2042                                           "VLAN cannot be empty");
2043         return 0;
2044 }
2045
2046 /**
2047  * Validate IPV4 item.
2048  *
2049  * @param[in] item
2050  *   Item specification.
2051  * @param[in] item_flags
2052  *   Bit-fields that holds the items detected until now.
2053  * @param[in] last_item
2054  *   Previous validated item in the pattern items.
2055  * @param[in] ether_type
2056  *   Type in the ethernet layer header (including dot1q).
2057  * @param[in] acc_mask
2058  *   Acceptable mask, if NULL default internal default mask
2059  *   will be used to check whether item fields are supported.
2060  * @param[in] range_accepted
2061  *   True if range of values is accepted for specific fields, false otherwise.
2062  * @param[out] error
2063  *   Pointer to error structure.
2064  *
2065  * @return
2066  *   0 on success, a negative errno value otherwise and rte_errno is set.
2067  */
2068 int
2069 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2070                              uint64_t item_flags,
2071                              uint64_t last_item,
2072                              uint16_t ether_type,
2073                              const struct rte_flow_item_ipv4 *acc_mask,
2074                              bool range_accepted,
2075                              struct rte_flow_error *error)
2076 {
2077         const struct rte_flow_item_ipv4 *mask = item->mask;
2078         const struct rte_flow_item_ipv4 *spec = item->spec;
2079         const struct rte_flow_item_ipv4 nic_mask = {
2080                 .hdr = {
2081                         .src_addr = RTE_BE32(0xffffffff),
2082                         .dst_addr = RTE_BE32(0xffffffff),
2083                         .type_of_service = 0xff,
2084                         .next_proto_id = 0xff,
2085                 },
2086         };
2087         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2088         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2089                                       MLX5_FLOW_LAYER_OUTER_L3;
2090         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2091                                       MLX5_FLOW_LAYER_OUTER_L4;
2092         int ret;
2093         uint8_t next_proto = 0xFF;
2094         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2095                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2096                                   MLX5_FLOW_LAYER_INNER_VLAN);
2097
2098         if ((last_item & l2_vlan) && ether_type &&
2099             ether_type != RTE_ETHER_TYPE_IPV4)
2100                 return rte_flow_error_set(error, EINVAL,
2101                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2102                                           "IPv4 cannot follow L2/VLAN layer "
2103                                           "which ether type is not IPv4");
2104         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2105                 if (mask && spec)
2106                         next_proto = mask->hdr.next_proto_id &
2107                                      spec->hdr.next_proto_id;
2108                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2109                         return rte_flow_error_set(error, EINVAL,
2110                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2111                                                   item,
2112                                                   "multiple tunnel "
2113                                                   "not supported");
2114         }
2115         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2116                 return rte_flow_error_set(error, EINVAL,
2117                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2118                                           "wrong tunnel type - IPv6 specified "
2119                                           "but IPv4 item provided");
2120         if (item_flags & l3m)
2121                 return rte_flow_error_set(error, ENOTSUP,
2122                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2123                                           "multiple L3 layers not supported");
2124         else if (item_flags & l4m)
2125                 return rte_flow_error_set(error, EINVAL,
2126                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2127                                           "L3 cannot follow an L4 layer.");
2128         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2129                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2130                 return rte_flow_error_set(error, EINVAL,
2131                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2132                                           "L3 cannot follow an NVGRE layer.");
2133         if (!mask)
2134                 mask = &rte_flow_item_ipv4_mask;
2135         else if (mask->hdr.next_proto_id != 0 &&
2136                  mask->hdr.next_proto_id != 0xff)
2137                 return rte_flow_error_set(error, EINVAL,
2138                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2139                                           "partial mask is not supported"
2140                                           " for protocol");
2141         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2142                                         acc_mask ? (const uint8_t *)acc_mask
2143                                                  : (const uint8_t *)&nic_mask,
2144                                         sizeof(struct rte_flow_item_ipv4),
2145                                         range_accepted, error);
2146         if (ret < 0)
2147                 return ret;
2148         return 0;
2149 }
2150
2151 /**
2152  * Validate IPV6 item.
2153  *
2154  * @param[in] item
2155  *   Item specification.
2156  * @param[in] item_flags
2157  *   Bit-fields that holds the items detected until now.
2158  * @param[in] last_item
2159  *   Previous validated item in the pattern items.
2160  * @param[in] ether_type
2161  *   Type in the ethernet layer header (including dot1q).
2162  * @param[in] acc_mask
2163  *   Acceptable mask, if NULL default internal default mask
2164  *   will be used to check whether item fields are supported.
2165  * @param[out] error
2166  *   Pointer to error structure.
2167  *
2168  * @return
2169  *   0 on success, a negative errno value otherwise and rte_errno is set.
2170  */
2171 int
2172 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2173                              uint64_t item_flags,
2174                              uint64_t last_item,
2175                              uint16_t ether_type,
2176                              const struct rte_flow_item_ipv6 *acc_mask,
2177                              struct rte_flow_error *error)
2178 {
2179         const struct rte_flow_item_ipv6 *mask = item->mask;
2180         const struct rte_flow_item_ipv6 *spec = item->spec;
2181         const struct rte_flow_item_ipv6 nic_mask = {
2182                 .hdr = {
2183                         .src_addr =
2184                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2185                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2186                         .dst_addr =
2187                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2188                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2189                         .vtc_flow = RTE_BE32(0xffffffff),
2190                         .proto = 0xff,
2191                 },
2192         };
2193         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2194         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2195                                       MLX5_FLOW_LAYER_OUTER_L3;
2196         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2197                                       MLX5_FLOW_LAYER_OUTER_L4;
2198         int ret;
2199         uint8_t next_proto = 0xFF;
2200         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2201                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2202                                   MLX5_FLOW_LAYER_INNER_VLAN);
2203
2204         if ((last_item & l2_vlan) && ether_type &&
2205             ether_type != RTE_ETHER_TYPE_IPV6)
2206                 return rte_flow_error_set(error, EINVAL,
2207                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2208                                           "IPv6 cannot follow L2/VLAN layer "
2209                                           "which ether type is not IPv6");
2210         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2211                 next_proto = spec->hdr.proto;
2212         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2213                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2214                         return rte_flow_error_set(error, EINVAL,
2215                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2216                                                   item,
2217                                                   "multiple tunnel "
2218                                                   "not supported");
2219         }
2220         if (next_proto == IPPROTO_HOPOPTS  ||
2221             next_proto == IPPROTO_ROUTING  ||
2222             next_proto == IPPROTO_FRAGMENT ||
2223             next_proto == IPPROTO_ESP      ||
2224             next_proto == IPPROTO_AH       ||
2225             next_proto == IPPROTO_DSTOPTS)
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2228                                           "IPv6 proto (next header) should "
2229                                           "not be set as extension header");
2230         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2231                 return rte_flow_error_set(error, EINVAL,
2232                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2233                                           "wrong tunnel type - IPv4 specified "
2234                                           "but IPv6 item provided");
2235         if (item_flags & l3m)
2236                 return rte_flow_error_set(error, ENOTSUP,
2237                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2238                                           "multiple L3 layers not supported");
2239         else if (item_flags & l4m)
2240                 return rte_flow_error_set(error, EINVAL,
2241                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2242                                           "L3 cannot follow an L4 layer.");
2243         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2244                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2245                 return rte_flow_error_set(error, EINVAL,
2246                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2247                                           "L3 cannot follow an NVGRE layer.");
2248         if (!mask)
2249                 mask = &rte_flow_item_ipv6_mask;
2250         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2251                                         acc_mask ? (const uint8_t *)acc_mask
2252                                                  : (const uint8_t *)&nic_mask,
2253                                         sizeof(struct rte_flow_item_ipv6),
2254                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2255         if (ret < 0)
2256                 return ret;
2257         return 0;
2258 }
2259
2260 /**
2261  * Validate UDP item.
2262  *
2263  * @param[in] item
2264  *   Item specification.
2265  * @param[in] item_flags
2266  *   Bit-fields that holds the items detected until now.
2267  * @param[in] target_protocol
2268  *   The next protocol in the previous item.
2269  * @param[in] flow_mask
2270  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2271  * @param[out] error
2272  *   Pointer to error structure.
2273  *
2274  * @return
2275  *   0 on success, a negative errno value otherwise and rte_errno is set.
2276  */
2277 int
2278 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2279                             uint64_t item_flags,
2280                             uint8_t target_protocol,
2281                             struct rte_flow_error *error)
2282 {
2283         const struct rte_flow_item_udp *mask = item->mask;
2284         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2285         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2286                                       MLX5_FLOW_LAYER_OUTER_L3;
2287         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2288                                       MLX5_FLOW_LAYER_OUTER_L4;
2289         int ret;
2290
2291         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2292                 return rte_flow_error_set(error, EINVAL,
2293                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2294                                           "protocol filtering not compatible"
2295                                           " with UDP layer");
2296         if (!(item_flags & l3m))
2297                 return rte_flow_error_set(error, EINVAL,
2298                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2299                                           "L3 is mandatory to filter on L4");
2300         if (item_flags & l4m)
2301                 return rte_flow_error_set(error, EINVAL,
2302                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2303                                           "multiple L4 layers not supported");
2304         if (!mask)
2305                 mask = &rte_flow_item_udp_mask;
2306         ret = mlx5_flow_item_acceptable
2307                 (item, (const uint8_t *)mask,
2308                  (const uint8_t *)&rte_flow_item_udp_mask,
2309                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2310                  error);
2311         if (ret < 0)
2312                 return ret;
2313         return 0;
2314 }
2315
2316 /**
2317  * Validate TCP item.
2318  *
2319  * @param[in] item
2320  *   Item specification.
2321  * @param[in] item_flags
2322  *   Bit-fields that holds the items detected until now.
2323  * @param[in] target_protocol
2324  *   The next protocol in the previous item.
2325  * @param[out] error
2326  *   Pointer to error structure.
2327  *
2328  * @return
2329  *   0 on success, a negative errno value otherwise and rte_errno is set.
2330  */
2331 int
2332 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2333                             uint64_t item_flags,
2334                             uint8_t target_protocol,
2335                             const struct rte_flow_item_tcp *flow_mask,
2336                             struct rte_flow_error *error)
2337 {
2338         const struct rte_flow_item_tcp *mask = item->mask;
2339         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2340         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2341                                       MLX5_FLOW_LAYER_OUTER_L3;
2342         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2343                                       MLX5_FLOW_LAYER_OUTER_L4;
2344         int ret;
2345
2346         MLX5_ASSERT(flow_mask);
2347         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2348                 return rte_flow_error_set(error, EINVAL,
2349                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2350                                           "protocol filtering not compatible"
2351                                           " with TCP layer");
2352         if (!(item_flags & l3m))
2353                 return rte_flow_error_set(error, EINVAL,
2354                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2355                                           "L3 is mandatory to filter on L4");
2356         if (item_flags & l4m)
2357                 return rte_flow_error_set(error, EINVAL,
2358                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2359                                           "multiple L4 layers not supported");
2360         if (!mask)
2361                 mask = &rte_flow_item_tcp_mask;
2362         ret = mlx5_flow_item_acceptable
2363                 (item, (const uint8_t *)mask,
2364                  (const uint8_t *)flow_mask,
2365                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2366                  error);
2367         if (ret < 0)
2368                 return ret;
2369         return 0;
2370 }
2371
2372 /**
2373  * Validate VXLAN item.
2374  *
2375  * @param[in] item
2376  *   Item specification.
2377  * @param[in] item_flags
2378  *   Bit-fields that holds the items detected until now.
2379  * @param[in] target_protocol
2380  *   The next protocol in the previous item.
2381  * @param[out] error
2382  *   Pointer to error structure.
2383  *
2384  * @return
2385  *   0 on success, a negative errno value otherwise and rte_errno is set.
2386  */
2387 int
2388 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2389                               uint64_t item_flags,
2390                               struct rte_flow_error *error)
2391 {
2392         const struct rte_flow_item_vxlan *spec = item->spec;
2393         const struct rte_flow_item_vxlan *mask = item->mask;
2394         int ret;
2395         union vni {
2396                 uint32_t vlan_id;
2397                 uint8_t vni[4];
2398         } id = { .vlan_id = 0, };
2399
2400
2401         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2402                 return rte_flow_error_set(error, ENOTSUP,
2403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2404                                           "multiple tunnel layers not"
2405                                           " supported");
2406         /*
2407          * Verify only UDPv4 is present as defined in
2408          * https://tools.ietf.org/html/rfc7348
2409          */
2410         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2411                 return rte_flow_error_set(error, EINVAL,
2412                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2413                                           "no outer UDP layer found");
2414         if (!mask)
2415                 mask = &rte_flow_item_vxlan_mask;
2416         ret = mlx5_flow_item_acceptable
2417                 (item, (const uint8_t *)mask,
2418                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2419                  sizeof(struct rte_flow_item_vxlan),
2420                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2421         if (ret < 0)
2422                 return ret;
2423         if (spec) {
2424                 memcpy(&id.vni[1], spec->vni, 3);
2425                 memcpy(&id.vni[1], mask->vni, 3);
2426         }
2427         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2428                 return rte_flow_error_set(error, ENOTSUP,
2429                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2430                                           "VXLAN tunnel must be fully defined");
2431         return 0;
2432 }
2433
2434 /**
2435  * Validate VXLAN_GPE item.
2436  *
2437  * @param[in] item
2438  *   Item specification.
2439  * @param[in] item_flags
2440  *   Bit-fields that holds the items detected until now.
2441  * @param[in] priv
2442  *   Pointer to the private data structure.
2443  * @param[in] target_protocol
2444  *   The next protocol in the previous item.
2445  * @param[out] error
2446  *   Pointer to error structure.
2447  *
2448  * @return
2449  *   0 on success, a negative errno value otherwise and rte_errno is set.
2450  */
2451 int
2452 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2453                                   uint64_t item_flags,
2454                                   struct rte_eth_dev *dev,
2455                                   struct rte_flow_error *error)
2456 {
2457         struct mlx5_priv *priv = dev->data->dev_private;
2458         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2459         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2460         int ret;
2461         union vni {
2462                 uint32_t vlan_id;
2463                 uint8_t vni[4];
2464         } id = { .vlan_id = 0, };
2465
2466         if (!priv->config.l3_vxlan_en)
2467                 return rte_flow_error_set(error, ENOTSUP,
2468                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2469                                           "L3 VXLAN is not enabled by device"
2470                                           " parameter and/or not configured in"
2471                                           " firmware");
2472         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2473                 return rte_flow_error_set(error, ENOTSUP,
2474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2475                                           "multiple tunnel layers not"
2476                                           " supported");
2477         /*
2478          * Verify only UDPv4 is present as defined in
2479          * https://tools.ietf.org/html/rfc7348
2480          */
2481         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2484                                           "no outer UDP layer found");
2485         if (!mask)
2486                 mask = &rte_flow_item_vxlan_gpe_mask;
2487         ret = mlx5_flow_item_acceptable
2488                 (item, (const uint8_t *)mask,
2489                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2490                  sizeof(struct rte_flow_item_vxlan_gpe),
2491                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2492         if (ret < 0)
2493                 return ret;
2494         if (spec) {
2495                 if (spec->protocol)
2496                         return rte_flow_error_set(error, ENOTSUP,
2497                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2498                                                   item,
2499                                                   "VxLAN-GPE protocol"
2500                                                   " not supported");
2501                 memcpy(&id.vni[1], spec->vni, 3);
2502                 memcpy(&id.vni[1], mask->vni, 3);
2503         }
2504         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2505                 return rte_flow_error_set(error, ENOTSUP,
2506                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2507                                           "VXLAN-GPE tunnel must be fully"
2508                                           " defined");
2509         return 0;
2510 }
2511 /**
2512  * Validate GRE Key item.
2513  *
2514  * @param[in] item
2515  *   Item specification.
2516  * @param[in] item_flags
2517  *   Bit flags to mark detected items.
2518  * @param[in] gre_item
2519  *   Pointer to gre_item
2520  * @param[out] error
2521  *   Pointer to error structure.
2522  *
2523  * @return
2524  *   0 on success, a negative errno value otherwise and rte_errno is set.
2525  */
2526 int
2527 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2528                                 uint64_t item_flags,
2529                                 const struct rte_flow_item *gre_item,
2530                                 struct rte_flow_error *error)
2531 {
2532         const rte_be32_t *mask = item->mask;
2533         int ret = 0;
2534         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2535         const struct rte_flow_item_gre *gre_spec;
2536         const struct rte_flow_item_gre *gre_mask;
2537
2538         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2541                                           "Multiple GRE key not support");
2542         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2543                 return rte_flow_error_set(error, ENOTSUP,
2544                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2545                                           "No preceding GRE header");
2546         if (item_flags & MLX5_FLOW_LAYER_INNER)
2547                 return rte_flow_error_set(error, ENOTSUP,
2548                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2549                                           "GRE key following a wrong item");
2550         gre_mask = gre_item->mask;
2551         if (!gre_mask)
2552                 gre_mask = &rte_flow_item_gre_mask;
2553         gre_spec = gre_item->spec;
2554         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2555                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2556                 return rte_flow_error_set(error, EINVAL,
2557                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2558                                           "Key bit must be on");
2559
2560         if (!mask)
2561                 mask = &gre_key_default_mask;
2562         ret = mlx5_flow_item_acceptable
2563                 (item, (const uint8_t *)mask,
2564                  (const uint8_t *)&gre_key_default_mask,
2565                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2566         return ret;
2567 }
2568
2569 /**
2570  * Validate GRE item.
2571  *
2572  * @param[in] item
2573  *   Item specification.
2574  * @param[in] item_flags
2575  *   Bit flags to mark detected items.
2576  * @param[in] target_protocol
2577  *   The next protocol in the previous item.
2578  * @param[out] error
2579  *   Pointer to error structure.
2580  *
2581  * @return
2582  *   0 on success, a negative errno value otherwise and rte_errno is set.
2583  */
2584 int
2585 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2586                             uint64_t item_flags,
2587                             uint8_t target_protocol,
2588                             struct rte_flow_error *error)
2589 {
2590         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2591         const struct rte_flow_item_gre *mask = item->mask;
2592         int ret;
2593         const struct rte_flow_item_gre nic_mask = {
2594                 .c_rsvd0_ver = RTE_BE16(0xB000),
2595                 .protocol = RTE_BE16(UINT16_MAX),
2596         };
2597
2598         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2599                 return rte_flow_error_set(error, EINVAL,
2600                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2601                                           "protocol filtering not compatible"
2602                                           " with this GRE layer");
2603         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2604                 return rte_flow_error_set(error, ENOTSUP,
2605                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2606                                           "multiple tunnel layers not"
2607                                           " supported");
2608         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2609                 return rte_flow_error_set(error, ENOTSUP,
2610                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2611                                           "L3 Layer is missing");
2612         if (!mask)
2613                 mask = &rte_flow_item_gre_mask;
2614         ret = mlx5_flow_item_acceptable
2615                 (item, (const uint8_t *)mask,
2616                  (const uint8_t *)&nic_mask,
2617                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2618                  error);
2619         if (ret < 0)
2620                 return ret;
2621 #ifndef HAVE_MLX5DV_DR
2622 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2623         if (spec && (spec->protocol & mask->protocol))
2624                 return rte_flow_error_set(error, ENOTSUP,
2625                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2626                                           "without MPLS support the"
2627                                           " specification cannot be used for"
2628                                           " filtering");
2629 #endif
2630 #endif
2631         return 0;
2632 }
2633
2634 /**
2635  * Validate Geneve item.
2636  *
2637  * @param[in] item
2638  *   Item specification.
2639  * @param[in] itemFlags
2640  *   Bit-fields that holds the items detected until now.
2641  * @param[in] enPriv
2642  *   Pointer to the private data structure.
2643  * @param[out] error
2644  *   Pointer to error structure.
2645  *
2646  * @return
2647  *   0 on success, a negative errno value otherwise and rte_errno is set.
2648  */
2649
2650 int
2651 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2652                                uint64_t item_flags,
2653                                struct rte_eth_dev *dev,
2654                                struct rte_flow_error *error)
2655 {
2656         struct mlx5_priv *priv = dev->data->dev_private;
2657         const struct rte_flow_item_geneve *spec = item->spec;
2658         const struct rte_flow_item_geneve *mask = item->mask;
2659         int ret;
2660         uint16_t gbhdr;
2661         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2662                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2663         const struct rte_flow_item_geneve nic_mask = {
2664                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2665                 .vni = "\xff\xff\xff",
2666                 .protocol = RTE_BE16(UINT16_MAX),
2667         };
2668
2669         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2670                 return rte_flow_error_set(error, ENOTSUP,
2671                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2672                                           "L3 Geneve is not enabled by device"
2673                                           " parameter and/or not configured in"
2674                                           " firmware");
2675         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2676                 return rte_flow_error_set(error, ENOTSUP,
2677                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2678                                           "multiple tunnel layers not"
2679                                           " supported");
2680         /*
2681          * Verify only UDPv4 is present as defined in
2682          * https://tools.ietf.org/html/rfc7348
2683          */
2684         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2685                 return rte_flow_error_set(error, EINVAL,
2686                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2687                                           "no outer UDP layer found");
2688         if (!mask)
2689                 mask = &rte_flow_item_geneve_mask;
2690         ret = mlx5_flow_item_acceptable
2691                                   (item, (const uint8_t *)mask,
2692                                    (const uint8_t *)&nic_mask,
2693                                    sizeof(struct rte_flow_item_geneve),
2694                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2695         if (ret)
2696                 return ret;
2697         if (spec) {
2698                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2699                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2700                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2701                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2702                         return rte_flow_error_set(error, ENOTSUP,
2703                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2704                                                   item,
2705                                                   "Geneve protocol unsupported"
2706                                                   " fields are being used");
2707                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2708                         return rte_flow_error_set
2709                                         (error, ENOTSUP,
2710                                          RTE_FLOW_ERROR_TYPE_ITEM,
2711                                          item,
2712                                          "Unsupported Geneve options length");
2713         }
2714         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2715                 return rte_flow_error_set
2716                                     (error, ENOTSUP,
2717                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2718                                      "Geneve tunnel must be fully defined");
2719         return 0;
2720 }
2721
2722 /**
2723  * Validate MPLS item.
2724  *
2725  * @param[in] dev
2726  *   Pointer to the rte_eth_dev structure.
2727  * @param[in] item
2728  *   Item specification.
2729  * @param[in] item_flags
2730  *   Bit-fields that holds the items detected until now.
2731  * @param[in] prev_layer
2732  *   The protocol layer indicated in previous item.
2733  * @param[out] error
2734  *   Pointer to error structure.
2735  *
2736  * @return
2737  *   0 on success, a negative errno value otherwise and rte_errno is set.
2738  */
2739 int
2740 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2741                              const struct rte_flow_item *item __rte_unused,
2742                              uint64_t item_flags __rte_unused,
2743                              uint64_t prev_layer __rte_unused,
2744                              struct rte_flow_error *error)
2745 {
2746 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2747         const struct rte_flow_item_mpls *mask = item->mask;
2748         struct mlx5_priv *priv = dev->data->dev_private;
2749         int ret;
2750
2751         if (!priv->config.mpls_en)
2752                 return rte_flow_error_set(error, ENOTSUP,
2753                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2754                                           "MPLS not supported or"
2755                                           " disabled in firmware"
2756                                           " configuration.");
2757         /* MPLS over IP, UDP, GRE is allowed */
2758         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2759                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2760                             MLX5_FLOW_LAYER_GRE)))
2761                 return rte_flow_error_set(error, EINVAL,
2762                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2763                                           "protocol filtering not compatible"
2764                                           " with MPLS layer");
2765         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2766         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2767             !(item_flags & MLX5_FLOW_LAYER_GRE))
2768                 return rte_flow_error_set(error, ENOTSUP,
2769                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2770                                           "multiple tunnel layers not"
2771                                           " supported");
2772         if (!mask)
2773                 mask = &rte_flow_item_mpls_mask;
2774         ret = mlx5_flow_item_acceptable
2775                 (item, (const uint8_t *)mask,
2776                  (const uint8_t *)&rte_flow_item_mpls_mask,
2777                  sizeof(struct rte_flow_item_mpls),
2778                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2779         if (ret < 0)
2780                 return ret;
2781         return 0;
2782 #else
2783         return rte_flow_error_set(error, ENOTSUP,
2784                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2785                                   "MPLS is not supported by Verbs, please"
2786                                   " update.");
2787 #endif
2788 }
2789
2790 /**
2791  * Validate NVGRE item.
2792  *
2793  * @param[in] item
2794  *   Item specification.
2795  * @param[in] item_flags
2796  *   Bit flags to mark detected items.
2797  * @param[in] target_protocol
2798  *   The next protocol in the previous item.
2799  * @param[out] error
2800  *   Pointer to error structure.
2801  *
2802  * @return
2803  *   0 on success, a negative errno value otherwise and rte_errno is set.
2804  */
2805 int
2806 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2807                               uint64_t item_flags,
2808                               uint8_t target_protocol,
2809                               struct rte_flow_error *error)
2810 {
2811         const struct rte_flow_item_nvgre *mask = item->mask;
2812         int ret;
2813
2814         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2815                 return rte_flow_error_set(error, EINVAL,
2816                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2817                                           "protocol filtering not compatible"
2818                                           " with this GRE layer");
2819         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2820                 return rte_flow_error_set(error, ENOTSUP,
2821                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2822                                           "multiple tunnel layers not"
2823                                           " supported");
2824         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2825                 return rte_flow_error_set(error, ENOTSUP,
2826                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2827                                           "L3 Layer is missing");
2828         if (!mask)
2829                 mask = &rte_flow_item_nvgre_mask;
2830         ret = mlx5_flow_item_acceptable
2831                 (item, (const uint8_t *)mask,
2832                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2833                  sizeof(struct rte_flow_item_nvgre),
2834                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2835         if (ret < 0)
2836                 return ret;
2837         return 0;
2838 }
2839
2840 /**
2841  * Validate eCPRI item.
2842  *
2843  * @param[in] item
2844  *   Item specification.
2845  * @param[in] item_flags
2846  *   Bit-fields that holds the items detected until now.
2847  * @param[in] last_item
2848  *   Previous validated item in the pattern items.
2849  * @param[in] ether_type
2850  *   Type in the ethernet layer header (including dot1q).
2851  * @param[in] acc_mask
2852  *   Acceptable mask, if NULL default internal default mask
2853  *   will be used to check whether item fields are supported.
2854  * @param[out] error
2855  *   Pointer to error structure.
2856  *
2857  * @return
2858  *   0 on success, a negative errno value otherwise and rte_errno is set.
2859  */
2860 int
2861 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2862                               uint64_t item_flags,
2863                               uint64_t last_item,
2864                               uint16_t ether_type,
2865                               const struct rte_flow_item_ecpri *acc_mask,
2866                               struct rte_flow_error *error)
2867 {
2868         const struct rte_flow_item_ecpri *mask = item->mask;
2869         const struct rte_flow_item_ecpri nic_mask = {
2870                 .hdr = {
2871                         .common = {
2872                                 .u32 =
2873                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2874                                         .type = 0xFF,
2875                                         }).u32),
2876                         },
2877                         .dummy[0] = 0xFFFFFFFF,
2878                 },
2879         };
2880         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2881                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2882         struct rte_flow_item_ecpri mask_lo;
2883
2884         if ((last_item & outer_l2_vlan) && ether_type &&
2885             ether_type != RTE_ETHER_TYPE_ECPRI)
2886                 return rte_flow_error_set(error, EINVAL,
2887                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2888                                           "eCPRI cannot follow L2/VLAN layer "
2889                                           "which ether type is not 0xAEFE.");
2890         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2891                 return rte_flow_error_set(error, EINVAL,
2892                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2893                                           "eCPRI with tunnel is not supported "
2894                                           "right now.");
2895         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2896                 return rte_flow_error_set(error, ENOTSUP,
2897                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2898                                           "multiple L3 layers not supported");
2899         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2900                 return rte_flow_error_set(error, EINVAL,
2901                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2902                                           "eCPRI cannot follow a TCP layer.");
2903         /* In specification, eCPRI could be over UDP layer. */
2904         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2905                 return rte_flow_error_set(error, EINVAL,
2906                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2907                                           "eCPRI over UDP layer is not yet "
2908                                           "supported right now.");
2909         /* Mask for type field in common header could be zero. */
2910         if (!mask)
2911                 mask = &rte_flow_item_ecpri_mask;
2912         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2913         /* Input mask is in big-endian format. */
2914         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2915                 return rte_flow_error_set(error, EINVAL,
2916                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2917                                           "partial mask is not supported "
2918                                           "for protocol");
2919         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2920                 return rte_flow_error_set(error, EINVAL,
2921                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2922                                           "message header mask must be after "
2923                                           "a type mask");
2924         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2925                                          acc_mask ? (const uint8_t *)acc_mask
2926                                                   : (const uint8_t *)&nic_mask,
2927                                          sizeof(struct rte_flow_item_ecpri),
2928                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2929 }
2930
2931 /* Allocate unique ID for the split Q/RSS subflows. */
2932 static uint32_t
2933 flow_qrss_get_id(struct rte_eth_dev *dev)
2934 {
2935         struct mlx5_priv *priv = dev->data->dev_private;
2936         uint32_t qrss_id, ret;
2937
2938         ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2939         if (ret)
2940                 return 0;
2941         MLX5_ASSERT(qrss_id);
2942         return qrss_id;
2943 }
2944
2945 /* Free unique ID for the split Q/RSS subflows. */
2946 static void
2947 flow_qrss_free_id(struct rte_eth_dev *dev,  uint32_t qrss_id)
2948 {
2949         struct mlx5_priv *priv = dev->data->dev_private;
2950
2951         if (qrss_id)
2952                 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2953 }
2954
2955 /**
2956  * Release resource related QUEUE/RSS action split.
2957  *
2958  * @param dev
2959  *   Pointer to Ethernet device.
2960  * @param flow
2961  *   Flow to release id's from.
2962  */
2963 static void
2964 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2965                              struct rte_flow *flow)
2966 {
2967         struct mlx5_priv *priv = dev->data->dev_private;
2968         uint32_t handle_idx;
2969         struct mlx5_flow_handle *dev_handle;
2970
2971         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2972                        handle_idx, dev_handle, next)
2973                 if (dev_handle->split_flow_id)
2974                         flow_qrss_free_id(dev, dev_handle->split_flow_id);
2975 }
2976
2977 static int
2978 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2979                    const struct rte_flow_attr *attr __rte_unused,
2980                    const struct rte_flow_item items[] __rte_unused,
2981                    const struct rte_flow_action actions[] __rte_unused,
2982                    bool external __rte_unused,
2983                    int hairpin __rte_unused,
2984                    struct rte_flow_error *error)
2985 {
2986         return rte_flow_error_set(error, ENOTSUP,
2987                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2988 }
2989
2990 static struct mlx5_flow *
2991 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2992                   const struct rte_flow_attr *attr __rte_unused,
2993                   const struct rte_flow_item items[] __rte_unused,
2994                   const struct rte_flow_action actions[] __rte_unused,
2995                   struct rte_flow_error *error)
2996 {
2997         rte_flow_error_set(error, ENOTSUP,
2998                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2999         return NULL;
3000 }
3001
3002 static int
3003 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3004                     struct mlx5_flow *dev_flow __rte_unused,
3005                     const struct rte_flow_attr *attr __rte_unused,
3006                     const struct rte_flow_item items[] __rte_unused,
3007                     const struct rte_flow_action actions[] __rte_unused,
3008                     struct rte_flow_error *error)
3009 {
3010         return rte_flow_error_set(error, ENOTSUP,
3011                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3012 }
3013
3014 static int
3015 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3016                 struct rte_flow *flow __rte_unused,
3017                 struct rte_flow_error *error)
3018 {
3019         return rte_flow_error_set(error, ENOTSUP,
3020                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3021 }
3022
3023 static void
3024 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3025                  struct rte_flow *flow __rte_unused)
3026 {
3027 }
3028
3029 static void
3030 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3031                   struct rte_flow *flow __rte_unused)
3032 {
3033 }
3034
3035 static int
3036 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3037                 struct rte_flow *flow __rte_unused,
3038                 const struct rte_flow_action *actions __rte_unused,
3039                 void *data __rte_unused,
3040                 struct rte_flow_error *error)
3041 {
3042         return rte_flow_error_set(error, ENOTSUP,
3043                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3044 }
3045
3046 static int
3047 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3048                       uint32_t domains __rte_unused,
3049                       uint32_t flags __rte_unused)
3050 {
3051         return 0;
3052 }
3053
3054 /* Void driver to protect from null pointer reference. */
3055 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3056         .validate = flow_null_validate,
3057         .prepare = flow_null_prepare,
3058         .translate = flow_null_translate,
3059         .apply = flow_null_apply,
3060         .remove = flow_null_remove,
3061         .destroy = flow_null_destroy,
3062         .query = flow_null_query,
3063         .sync_domain = flow_null_sync_domain,
3064 };
3065
3066 /**
3067  * Select flow driver type according to flow attributes and device
3068  * configuration.
3069  *
3070  * @param[in] dev
3071  *   Pointer to the dev structure.
3072  * @param[in] attr
3073  *   Pointer to the flow attributes.
3074  *
3075  * @return
3076  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3077  */
3078 static enum mlx5_flow_drv_type
3079 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3080 {
3081         struct mlx5_priv *priv = dev->data->dev_private;
3082         /* The OS can determine first a specific flow type (DV, VERBS) */
3083         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3084
3085         if (type != MLX5_FLOW_TYPE_MAX)
3086                 return type;
3087         /* If no OS specific type - continue with DV/VERBS selection */
3088         if (attr->transfer && priv->config.dv_esw_en)
3089                 type = MLX5_FLOW_TYPE_DV;
3090         if (!attr->transfer)
3091                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3092                                                  MLX5_FLOW_TYPE_VERBS;
3093         return type;
3094 }
3095
3096 #define flow_get_drv_ops(type) flow_drv_ops[type]
3097
3098 /**
3099  * Flow driver validation API. This abstracts calling driver specific functions.
3100  * The type of flow driver is determined according to flow attributes.
3101  *
3102  * @param[in] dev
3103  *   Pointer to the dev structure.
3104  * @param[in] attr
3105  *   Pointer to the flow attributes.
3106  * @param[in] items
3107  *   Pointer to the list of items.
3108  * @param[in] actions
3109  *   Pointer to the list of actions.
3110  * @param[in] external
3111  *   This flow rule is created by request external to PMD.
3112  * @param[in] hairpin
3113  *   Number of hairpin TX actions, 0 means classic flow.
3114  * @param[out] error
3115  *   Pointer to the error structure.
3116  *
3117  * @return
3118  *   0 on success, a negative errno value otherwise and rte_errno is set.
3119  */
3120 static inline int
3121 flow_drv_validate(struct rte_eth_dev *dev,
3122                   const struct rte_flow_attr *attr,
3123                   const struct rte_flow_item items[],
3124                   const struct rte_flow_action actions[],
3125                   bool external, int hairpin, struct rte_flow_error *error)
3126 {
3127         const struct mlx5_flow_driver_ops *fops;
3128         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3129
3130         fops = flow_get_drv_ops(type);
3131         return fops->validate(dev, attr, items, actions, external,
3132                               hairpin, error);
3133 }
3134
3135 /**
3136  * Flow driver preparation API. This abstracts calling driver specific
3137  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3138  * calculates the size of memory required for device flow, allocates the memory,
3139  * initializes the device flow and returns the pointer.
3140  *
3141  * @note
3142  *   This function initializes device flow structure such as dv or verbs in
3143  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3144  *   rest. For example, adding returning device flow to flow->dev_flow list and
3145  *   setting backward reference to the flow should be done out of this function.
3146  *   layers field is not filled either.
3147  *
3148  * @param[in] dev
3149  *   Pointer to the dev structure.
3150  * @param[in] attr
3151  *   Pointer to the flow attributes.
3152  * @param[in] items
3153  *   Pointer to the list of items.
3154  * @param[in] actions
3155  *   Pointer to the list of actions.
3156  * @param[in] flow_idx
3157  *   This memory pool index to the flow.
3158  * @param[out] error
3159  *   Pointer to the error structure.
3160  *
3161  * @return
3162  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3163  */
3164 static inline struct mlx5_flow *
3165 flow_drv_prepare(struct rte_eth_dev *dev,
3166                  const struct rte_flow *flow,
3167                  const struct rte_flow_attr *attr,
3168                  const struct rte_flow_item items[],
3169                  const struct rte_flow_action actions[],
3170                  uint32_t flow_idx,
3171                  struct rte_flow_error *error)
3172 {
3173         const struct mlx5_flow_driver_ops *fops;
3174         enum mlx5_flow_drv_type type = flow->drv_type;
3175         struct mlx5_flow *mlx5_flow = NULL;
3176
3177         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3178         fops = flow_get_drv_ops(type);
3179         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3180         if (mlx5_flow)
3181                 mlx5_flow->flow_idx = flow_idx;
3182         return mlx5_flow;
3183 }
3184
3185 /**
3186  * Flow driver translation API. This abstracts calling driver specific
3187  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3188  * translates a generic flow into a driver flow. flow_drv_prepare() must
3189  * precede.
3190  *
3191  * @note
3192  *   dev_flow->layers could be filled as a result of parsing during translation
3193  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3194  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3195  *   flow->actions could be overwritten even though all the expanded dev_flows
3196  *   have the same actions.
3197  *
3198  * @param[in] dev
3199  *   Pointer to the rte dev structure.
3200  * @param[in, out] dev_flow
3201  *   Pointer to the mlx5 flow.
3202  * @param[in] attr
3203  *   Pointer to the flow attributes.
3204  * @param[in] items
3205  *   Pointer to the list of items.
3206  * @param[in] actions
3207  *   Pointer to the list of actions.
3208  * @param[out] error
3209  *   Pointer to the error structure.
3210  *
3211  * @return
3212  *   0 on success, a negative errno value otherwise and rte_errno is set.
3213  */
3214 static inline int
3215 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3216                    const struct rte_flow_attr *attr,
3217                    const struct rte_flow_item items[],
3218                    const struct rte_flow_action actions[],
3219                    struct rte_flow_error *error)
3220 {
3221         const struct mlx5_flow_driver_ops *fops;
3222         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3223
3224         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3225         fops = flow_get_drv_ops(type);
3226         return fops->translate(dev, dev_flow, attr, items, actions, error);
3227 }
3228
3229 /**
3230  * Flow driver apply API. This abstracts calling driver specific functions.
3231  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3232  * translated driver flows on to device. flow_drv_translate() must precede.
3233  *
3234  * @param[in] dev
3235  *   Pointer to Ethernet device structure.
3236  * @param[in, out] flow
3237  *   Pointer to flow structure.
3238  * @param[out] error
3239  *   Pointer to error structure.
3240  *
3241  * @return
3242  *   0 on success, a negative errno value otherwise and rte_errno is set.
3243  */
3244 static inline int
3245 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3246                struct rte_flow_error *error)
3247 {
3248         const struct mlx5_flow_driver_ops *fops;
3249         enum mlx5_flow_drv_type type = flow->drv_type;
3250
3251         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3252         fops = flow_get_drv_ops(type);
3253         return fops->apply(dev, flow, error);
3254 }
3255
3256 /**
3257  * Flow driver remove API. This abstracts calling driver specific functions.
3258  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3259  * on device. All the resources of the flow should be freed by calling
3260  * flow_drv_destroy().
3261  *
3262  * @param[in] dev
3263  *   Pointer to Ethernet device.
3264  * @param[in, out] flow
3265  *   Pointer to flow structure.
3266  */
3267 static inline void
3268 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
3269 {
3270         const struct mlx5_flow_driver_ops *fops;
3271         enum mlx5_flow_drv_type type = flow->drv_type;
3272
3273         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3274         fops = flow_get_drv_ops(type);
3275         fops->remove(dev, flow);
3276 }
3277
3278 /**
3279  * Flow driver destroy API. This abstracts calling driver specific functions.
3280  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3281  * on device and releases resources of the flow.
3282  *
3283  * @param[in] dev
3284  *   Pointer to Ethernet device.
3285  * @param[in, out] flow
3286  *   Pointer to flow structure.
3287  */
3288 static inline void
3289 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3290 {
3291         const struct mlx5_flow_driver_ops *fops;
3292         enum mlx5_flow_drv_type type = flow->drv_type;
3293
3294         flow_mreg_split_qrss_release(dev, flow);
3295         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3296         fops = flow_get_drv_ops(type);
3297         fops->destroy(dev, flow);
3298 }
3299
3300 /**
3301  * Get RSS action from the action list.
3302  *
3303  * @param[in] actions
3304  *   Pointer to the list of actions.
3305  *
3306  * @return
3307  *   Pointer to the RSS action if exist, else return NULL.
3308  */
3309 static const struct rte_flow_action_rss*
3310 flow_get_rss_action(const struct rte_flow_action actions[])
3311 {
3312         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3313                 switch (actions->type) {
3314                 case RTE_FLOW_ACTION_TYPE_RSS:
3315                         return (const struct rte_flow_action_rss *)
3316                                actions->conf;
3317                 default:
3318                         break;
3319                 }
3320         }
3321         return NULL;
3322 }
3323
3324 /* maps shared action to translated non shared in some actions array */
3325 struct mlx5_translated_shared_action {
3326         struct rte_flow_shared_action *action; /**< Shared action */
3327         int index; /**< Index in related array of rte_flow_action */
3328 };
3329
3330 /**
3331  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3332  * non shared action if translation possible.
3333  * This functionality used to run same execution path for both shared & non
3334  * shared actions on flow create. All necessary preparations for shared
3335  * action handling should be preformed on *shared* actions list returned
3336  * from this call.
3337  *
3338  * @param[in] actions
3339  *   List of actions to translate.
3340  * @param[out] shared
3341  *   List to store translated shared actions.
3342  * @param[in, out] shared_n
3343  *   Size of *shared* array. On return should be updated with number of shared
3344  *   actions retrieved from the *actions* list.
3345  * @param[out] translated_actions
3346  *   List of actions where all shared actions were translated to non shared
3347  *   if possible. NULL if no translation took place.
3348  * @param[out] error
3349  *   Pointer to the error structure.
3350  *
3351  * @return
3352  *   0 on success, a negative errno value otherwise and rte_errno is set.
3353  */
3354 static int
3355 flow_shared_actions_translate(const struct rte_flow_action actions[],
3356         struct mlx5_translated_shared_action *shared,
3357         int *shared_n,
3358         struct rte_flow_action **translated_actions,
3359         struct rte_flow_error *error)
3360 {
3361         struct rte_flow_action *translated = NULL;
3362         size_t actions_size;
3363         int n;
3364         int copied_n = 0;
3365         struct mlx5_translated_shared_action *shared_end = NULL;
3366
3367         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3368                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3369                         continue;
3370                 if (copied_n == *shared_n) {
3371                         return rte_flow_error_set
3372                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3373                                  NULL, "too many shared actions");
3374                 }
3375                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3376                            sizeof(actions[n].conf));
3377                 shared[copied_n].index = n;
3378                 copied_n++;
3379         }
3380         n++;
3381         *shared_n = copied_n;
3382         if (!copied_n)
3383                 return 0;
3384         actions_size = sizeof(struct rte_flow_action) * n;
3385         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3386         if (!translated) {
3387                 rte_errno = ENOMEM;
3388                 return -ENOMEM;
3389         }
3390         memcpy(translated, actions, actions_size);
3391         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3392                 const struct rte_flow_shared_action *shared_action;
3393
3394                 shared_action = shared->action;
3395                 switch (shared_action->type) {
3396                 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
3397                         translated[shared->index].type =
3398                                 RTE_FLOW_ACTION_TYPE_RSS;
3399                         translated[shared->index].conf =
3400                                 &shared_action->rss.origin;
3401                         break;
3402                 default:
3403                         mlx5_free(translated);
3404                         return rte_flow_error_set
3405                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3406                                  NULL, "invalid shared action type");
3407                 }
3408         }
3409         *translated_actions = translated;
3410         return 0;
3411 }
3412
3413 /**
3414  * Get Shared RSS action from the action list.
3415  *
3416  * @param[in] shared
3417  *   Pointer to the list of actions.
3418  * @param[in] shared_n
3419  *   Actions list length.
3420  *
3421  * @return
3422  *   Pointer to the MLX5 RSS action if exists, otherwise return NULL.
3423  */
3424 static struct mlx5_shared_action_rss *
3425 flow_get_shared_rss_action(struct mlx5_translated_shared_action *shared,
3426                            int shared_n)
3427 {
3428         struct mlx5_translated_shared_action *shared_end;
3429
3430         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3431                 struct rte_flow_shared_action *shared_action;
3432
3433                 shared_action = shared->action;
3434                 switch (shared_action->type) {
3435                 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
3436                         __atomic_add_fetch(&shared_action->refcnt, 1,
3437                                            __ATOMIC_RELAXED);
3438                         return &shared_action->rss;
3439                 default:
3440                         break;
3441                 }
3442         }
3443         return NULL;
3444 }
3445
3446 struct rte_flow_shared_action *
3447 mlx5_flow_get_shared_rss(struct rte_flow *flow)
3448 {
3449         if (flow->shared_rss)
3450                 return container_of(flow->shared_rss,
3451                                     struct rte_flow_shared_action, rss);
3452         else
3453                 return NULL;
3454 }
3455
3456 static unsigned int
3457 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3458 {
3459         const struct rte_flow_item *item;
3460         unsigned int has_vlan = 0;
3461
3462         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3463                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3464                         has_vlan = 1;
3465                         break;
3466                 }
3467         }
3468         if (has_vlan)
3469                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3470                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3471         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3472                                MLX5_EXPANSION_ROOT_OUTER;
3473 }
3474
3475 /**
3476  *  Get layer flags from the prefix flow.
3477  *
3478  *  Some flows may be split to several subflows, the prefix subflow gets the
3479  *  match items and the suffix sub flow gets the actions.
3480  *  Some actions need the user defined match item flags to get the detail for
3481  *  the action.
3482  *  This function helps the suffix flow to get the item layer flags from prefix
3483  *  subflow.
3484  *
3485  * @param[in] dev_flow
3486  *   Pointer the created preifx subflow.
3487  *
3488  * @return
3489  *   The layers get from prefix subflow.
3490  */
3491 static inline uint64_t
3492 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3493 {
3494         uint64_t layers = 0;
3495
3496         /*
3497          * Layers bits could be localization, but usually the compiler will
3498          * help to do the optimization work for source code.
3499          * If no decap actions, use the layers directly.
3500          */
3501         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3502                 return dev_flow->handle->layers;
3503         /* Convert L3 layers with decap action. */
3504         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3505                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3506         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3507                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3508         /* Convert L4 layers with decap action.  */
3509         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3510                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3511         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3512                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3513         return layers;
3514 }
3515
3516 /**
3517  * Get metadata split action information.
3518  *
3519  * @param[in] actions
3520  *   Pointer to the list of actions.
3521  * @param[out] qrss
3522  *   Pointer to the return pointer.
3523  * @param[out] qrss_type
3524  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3525  *   if no QUEUE/RSS is found.
3526  * @param[out] encap_idx
3527  *   Pointer to the index of the encap action if exists, otherwise the last
3528  *   action index.
3529  *
3530  * @return
3531  *   Total number of actions.
3532  */
3533 static int
3534 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3535                                        const struct rte_flow_action **qrss,
3536                                        int *encap_idx)
3537 {
3538         const struct rte_flow_action_raw_encap *raw_encap;
3539         int actions_n = 0;
3540         int raw_decap_idx = -1;
3541
3542         *encap_idx = -1;
3543         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3544                 switch (actions->type) {
3545                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3546                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3547                         *encap_idx = actions_n;
3548                         break;
3549                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3550                         raw_decap_idx = actions_n;
3551                         break;
3552                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3553                         raw_encap = actions->conf;
3554                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3555                                 *encap_idx = raw_decap_idx != -1 ?
3556                                                       raw_decap_idx : actions_n;
3557                         break;
3558                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3559                 case RTE_FLOW_ACTION_TYPE_RSS:
3560                         *qrss = actions;
3561                         break;
3562                 default:
3563                         break;
3564                 }
3565                 actions_n++;
3566         }
3567         if (*encap_idx == -1)
3568                 *encap_idx = actions_n;
3569         /* Count RTE_FLOW_ACTION_TYPE_END. */
3570         return actions_n + 1;
3571 }
3572
3573 /**
3574  * Check meter action from the action list.
3575  *
3576  * @param[in] actions
3577  *   Pointer to the list of actions.
3578  * @param[out] mtr
3579  *   Pointer to the meter exist flag.
3580  *
3581  * @return
3582  *   Total number of actions.
3583  */
3584 static int
3585 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3586 {
3587         int actions_n = 0;
3588
3589         MLX5_ASSERT(mtr);
3590         *mtr = 0;
3591         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3592                 switch (actions->type) {
3593                 case RTE_FLOW_ACTION_TYPE_METER:
3594                         *mtr = 1;
3595                         break;
3596                 default:
3597                         break;
3598                 }
3599                 actions_n++;
3600         }
3601         /* Count RTE_FLOW_ACTION_TYPE_END. */
3602         return actions_n + 1;
3603 }
3604
3605 /**
3606  * Check if the flow should be split due to hairpin.
3607  * The reason for the split is that in current HW we can't
3608  * support encap and push-vlan on Rx, so if a flow contains
3609  * these actions we move it to Tx.
3610  *
3611  * @param dev
3612  *   Pointer to Ethernet device.
3613  * @param[in] attr
3614  *   Flow rule attributes.
3615  * @param[in] actions
3616  *   Associated actions (list terminated by the END action).
3617  *
3618  * @return
3619  *   > 0 the number of actions and the flow should be split,
3620  *   0 when no split required.
3621  */
3622 static int
3623 flow_check_hairpin_split(struct rte_eth_dev *dev,
3624                          const struct rte_flow_attr *attr,
3625                          const struct rte_flow_action actions[])
3626 {
3627         int queue_action = 0;
3628         int action_n = 0;
3629         int split = 0;
3630         const struct rte_flow_action_queue *queue;
3631         const struct rte_flow_action_rss *rss;
3632         const struct rte_flow_action_raw_encap *raw_encap;
3633         const struct rte_eth_hairpin_conf *conf;
3634
3635         if (!attr->ingress)
3636                 return 0;
3637         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3638                 switch (actions->type) {
3639                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3640                         queue = actions->conf;
3641                         if (queue == NULL)
3642                                 return 0;
3643                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3644                         if (conf != NULL && !!conf->tx_explicit)
3645                                 return 0;
3646                         queue_action = 1;
3647                         action_n++;
3648                         break;
3649                 case RTE_FLOW_ACTION_TYPE_RSS:
3650                         rss = actions->conf;
3651                         if (rss == NULL || rss->queue_num == 0)
3652                                 return 0;
3653                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3654                         if (conf != NULL && !!conf->tx_explicit)
3655                                 return 0;
3656                         queue_action = 1;
3657                         action_n++;
3658                         break;
3659                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3660                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3661                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3662                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3663                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3664                         split++;
3665                         action_n++;
3666                         break;
3667                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3668                         raw_encap = actions->conf;
3669                         if (raw_encap->size >
3670                             (sizeof(struct rte_flow_item_eth) +
3671                              sizeof(struct rte_flow_item_ipv4)))
3672                                 split++;
3673                         action_n++;
3674                         break;
3675                 default:
3676                         action_n++;
3677                         break;
3678                 }
3679         }
3680         if (split && queue_action)
3681                 return action_n;
3682         return 0;
3683 }
3684
3685 /* Declare flow create/destroy prototype in advance. */
3686 static uint32_t
3687 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3688                  const struct rte_flow_attr *attr,
3689                  const struct rte_flow_item items[],
3690                  const struct rte_flow_action actions[],
3691                  bool external, struct rte_flow_error *error);
3692
3693 static void
3694 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3695                   uint32_t flow_idx);
3696
3697 /**
3698  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3699  *
3700  * As mark_id is unique, if there's already a registered flow for the mark_id,
3701  * return by increasing the reference counter of the resource. Otherwise, create
3702  * the resource (mcp_res) and flow.
3703  *
3704  * Flow looks like,
3705  *   - If ingress port is ANY and reg_c[1] is mark_id,
3706  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3707  *
3708  * For default flow (zero mark_id), flow is like,
3709  *   - If ingress port is ANY,
3710  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3711  *
3712  * @param dev
3713  *   Pointer to Ethernet device.
3714  * @param mark_id
3715  *   ID of MARK action, zero means default flow for META.
3716  * @param[out] error
3717  *   Perform verbose error reporting if not NULL.
3718  *
3719  * @return
3720  *   Associated resource on success, NULL otherwise and rte_errno is set.
3721  */
3722 static struct mlx5_flow_mreg_copy_resource *
3723 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3724                           struct rte_flow_error *error)
3725 {
3726         struct mlx5_priv *priv = dev->data->dev_private;
3727         struct rte_flow_attr attr = {
3728                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3729                 .ingress = 1,
3730         };
3731         struct mlx5_rte_flow_item_tag tag_spec = {
3732                 .data = mark_id,
3733         };
3734         struct rte_flow_item items[] = {
3735                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3736         };
3737         struct rte_flow_action_mark ftag = {
3738                 .id = mark_id,
3739         };
3740         struct mlx5_flow_action_copy_mreg cp_mreg = {
3741                 .dst = REG_B,
3742                 .src = REG_NON,
3743         };
3744         struct rte_flow_action_jump jump = {
3745                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3746         };
3747         struct rte_flow_action actions[] = {
3748                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3749         };
3750         struct mlx5_flow_mreg_copy_resource *mcp_res;
3751         uint32_t idx = 0;
3752         int ret;
3753
3754         /* Fill the register fileds in the flow. */
3755         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3756         if (ret < 0)
3757                 return NULL;
3758         tag_spec.id = ret;
3759         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3760         if (ret < 0)
3761                 return NULL;
3762         cp_mreg.src = ret;
3763         /* Check if already registered. */
3764         MLX5_ASSERT(priv->mreg_cp_tbl);
3765         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
3766         if (mcp_res) {
3767                 /* For non-default rule. */
3768                 if (mark_id != MLX5_DEFAULT_COPY_ID)
3769                         mcp_res->refcnt++;
3770                 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
3771                             mcp_res->refcnt == 1);
3772                 return mcp_res;
3773         }
3774         /* Provide the full width of FLAG specific value. */
3775         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3776                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3777         /* Build a new flow. */
3778         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3779                 items[0] = (struct rte_flow_item){
3780                         .type = (enum rte_flow_item_type)
3781                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3782                         .spec = &tag_spec,
3783                 };
3784                 items[1] = (struct rte_flow_item){
3785                         .type = RTE_FLOW_ITEM_TYPE_END,
3786                 };
3787                 actions[0] = (struct rte_flow_action){
3788                         .type = (enum rte_flow_action_type)
3789                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3790                         .conf = &ftag,
3791                 };
3792                 actions[1] = (struct rte_flow_action){
3793                         .type = (enum rte_flow_action_type)
3794                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3795                         .conf = &cp_mreg,
3796                 };
3797                 actions[2] = (struct rte_flow_action){
3798                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3799                         .conf = &jump,
3800                 };
3801                 actions[3] = (struct rte_flow_action){
3802                         .type = RTE_FLOW_ACTION_TYPE_END,
3803                 };
3804         } else {
3805                 /* Default rule, wildcard match. */
3806                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3807                 items[0] = (struct rte_flow_item){
3808                         .type = RTE_FLOW_ITEM_TYPE_END,
3809                 };
3810                 actions[0] = (struct rte_flow_action){
3811                         .type = (enum rte_flow_action_type)
3812                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3813                         .conf = &cp_mreg,
3814                 };
3815                 actions[1] = (struct rte_flow_action){
3816                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3817                         .conf = &jump,
3818                 };
3819                 actions[2] = (struct rte_flow_action){
3820                         .type = RTE_FLOW_ACTION_TYPE_END,
3821                 };
3822         }
3823         /* Build a new entry. */
3824         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3825         if (!mcp_res) {
3826                 rte_errno = ENOMEM;
3827                 return NULL;
3828         }
3829         mcp_res->idx = idx;
3830         /*
3831          * The copy Flows are not included in any list. There
3832          * ones are referenced from other Flows and can not
3833          * be applied, removed, deleted in ardbitrary order
3834          * by list traversing.
3835          */
3836         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3837                                          actions, false, error);
3838         if (!mcp_res->rix_flow)
3839                 goto error;
3840         mcp_res->refcnt++;
3841         mcp_res->hlist_ent.key = mark_id;
3842         ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3843                                 &mcp_res->hlist_ent);
3844         MLX5_ASSERT(!ret);
3845         if (ret)
3846                 goto error;
3847         return mcp_res;
3848 error:
3849         if (mcp_res->rix_flow)
3850                 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3851         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3852         return NULL;
3853 }
3854
3855 /**
3856  * Release flow in RX_CP_TBL.
3857  *
3858  * @param dev
3859  *   Pointer to Ethernet device.
3860  * @flow
3861  *   Parent flow for wich copying is provided.
3862  */
3863 static void
3864 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3865                           struct rte_flow *flow)
3866 {
3867         struct mlx5_flow_mreg_copy_resource *mcp_res;
3868         struct mlx5_priv *priv = dev->data->dev_private;
3869
3870         if (!flow->rix_mreg_copy)
3871                 return;
3872         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3873                                  flow->rix_mreg_copy);
3874         if (!mcp_res || !priv->mreg_cp_tbl)
3875                 return;
3876         if (flow->copy_applied) {
3877                 MLX5_ASSERT(mcp_res->appcnt);
3878                 flow->copy_applied = 0;
3879                 --mcp_res->appcnt;
3880                 if (!mcp_res->appcnt) {
3881                         struct rte_flow *mcp_flow = mlx5_ipool_get
3882                                         (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3883                                         mcp_res->rix_flow);
3884
3885                         if (mcp_flow)
3886                                 flow_drv_remove(dev, mcp_flow);
3887                 }
3888         }
3889         /*
3890          * We do not check availability of metadata registers here,
3891          * because copy resources are not allocated in this case.
3892          */
3893         if (--mcp_res->refcnt)
3894                 return;
3895         MLX5_ASSERT(mcp_res->rix_flow);
3896         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3897         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3898         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3899         flow->rix_mreg_copy = 0;
3900 }
3901
3902 /**
3903  * Start flow in RX_CP_TBL.
3904  *
3905  * @param dev
3906  *   Pointer to Ethernet device.
3907  * @flow
3908  *   Parent flow for wich copying is provided.
3909  *
3910  * @return
3911  *   0 on success, a negative errno value otherwise and rte_errno is set.
3912  */
3913 static int
3914 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3915                             struct rte_flow *flow)
3916 {
3917         struct mlx5_flow_mreg_copy_resource *mcp_res;
3918         struct mlx5_priv *priv = dev->data->dev_private;
3919         int ret;
3920
3921         if (!flow->rix_mreg_copy || flow->copy_applied)
3922                 return 0;
3923         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3924                                  flow->rix_mreg_copy);
3925         if (!mcp_res)
3926                 return 0;
3927         if (!mcp_res->appcnt) {
3928                 struct rte_flow *mcp_flow = mlx5_ipool_get
3929                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3930                                 mcp_res->rix_flow);
3931
3932                 if (mcp_flow) {
3933                         ret = flow_drv_apply(dev, mcp_flow, NULL);
3934                         if (ret)
3935                                 return ret;
3936                 }
3937         }
3938         ++mcp_res->appcnt;
3939         flow->copy_applied = 1;
3940         return 0;
3941 }
3942
3943 /**
3944  * Stop flow in RX_CP_TBL.
3945  *
3946  * @param dev
3947  *   Pointer to Ethernet device.
3948  * @flow
3949  *   Parent flow for wich copying is provided.
3950  */
3951 static void
3952 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3953                            struct rte_flow *flow)
3954 {
3955         struct mlx5_flow_mreg_copy_resource *mcp_res;
3956         struct mlx5_priv *priv = dev->data->dev_private;
3957
3958         if (!flow->rix_mreg_copy || !flow->copy_applied)
3959                 return;
3960         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3961                                  flow->rix_mreg_copy);
3962         if (!mcp_res)
3963                 return;
3964         MLX5_ASSERT(mcp_res->appcnt);
3965         --mcp_res->appcnt;
3966         flow->copy_applied = 0;
3967         if (!mcp_res->appcnt) {
3968                 struct rte_flow *mcp_flow = mlx5_ipool_get
3969                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3970                                 mcp_res->rix_flow);
3971
3972                 if (mcp_flow)
3973                         flow_drv_remove(dev, mcp_flow);
3974         }
3975 }
3976
3977 /**
3978  * Remove the default copy action from RX_CP_TBL.
3979  *
3980  * @param dev
3981  *   Pointer to Ethernet device.
3982  */
3983 static void
3984 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3985 {
3986         struct mlx5_flow_mreg_copy_resource *mcp_res;
3987         struct mlx5_priv *priv = dev->data->dev_private;
3988
3989         /* Check if default flow is registered. */
3990         if (!priv->mreg_cp_tbl)
3991                 return;
3992         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3993                                             MLX5_DEFAULT_COPY_ID);
3994         if (!mcp_res)
3995                 return;
3996         MLX5_ASSERT(mcp_res->rix_flow);
3997         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3998         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3999         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4000 }
4001
4002 /**
4003  * Add the default copy action in in RX_CP_TBL.
4004  *
4005  * @param dev
4006  *   Pointer to Ethernet device.
4007  * @param[out] error
4008  *   Perform verbose error reporting if not NULL.
4009  *
4010  * @return
4011  *   0 for success, negative value otherwise and rte_errno is set.
4012  */
4013 static int
4014 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4015                                   struct rte_flow_error *error)
4016 {
4017         struct mlx5_priv *priv = dev->data->dev_private;
4018         struct mlx5_flow_mreg_copy_resource *mcp_res;
4019
4020         /* Check whether extensive metadata feature is engaged. */
4021         if (!priv->config.dv_flow_en ||
4022             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4023             !mlx5_flow_ext_mreg_supported(dev) ||
4024             !priv->sh->dv_regc0_mask)
4025                 return 0;
4026         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4027         if (!mcp_res)
4028                 return -rte_errno;
4029         return 0;
4030 }
4031
4032 /**
4033  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4034  *
4035  * All the flow having Q/RSS action should be split by
4036  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4037  * performs the following,
4038  *   - CQE->flow_tag := reg_c[1] (MARK)
4039  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4040  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4041  * but there should be a flow per each MARK ID set by MARK action.
4042  *
4043  * For the aforementioned reason, if there's a MARK action in flow's action
4044  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4045  * the MARK ID to CQE's flow_tag like,
4046  *   - If reg_c[1] is mark_id,
4047  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4048  *
4049  * For SET_META action which stores value in reg_c[0], as the destination is
4050  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4051  * MARK ID means the default flow. The default flow looks like,
4052  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4053  *
4054  * @param dev
4055  *   Pointer to Ethernet device.
4056  * @param flow
4057  *   Pointer to flow structure.
4058  * @param[in] actions
4059  *   Pointer to the list of actions.
4060  * @param[out] error
4061  *   Perform verbose error reporting if not NULL.
4062  *
4063  * @return
4064  *   0 on success, negative value otherwise and rte_errno is set.
4065  */
4066 static int
4067 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4068                             struct rte_flow *flow,
4069                             const struct rte_flow_action *actions,
4070                             struct rte_flow_error *error)
4071 {
4072         struct mlx5_priv *priv = dev->data->dev_private;
4073         struct mlx5_dev_config *config = &priv->config;
4074         struct mlx5_flow_mreg_copy_resource *mcp_res;
4075         const struct rte_flow_action_mark *mark;
4076
4077         /* Check whether extensive metadata feature is engaged. */
4078         if (!config->dv_flow_en ||
4079             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4080             !mlx5_flow_ext_mreg_supported(dev) ||
4081             !priv->sh->dv_regc0_mask)
4082                 return 0;
4083         /* Find MARK action. */
4084         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4085                 switch (actions->type) {
4086                 case RTE_FLOW_ACTION_TYPE_FLAG:
4087                         mcp_res = flow_mreg_add_copy_action
4088                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4089                         if (!mcp_res)
4090                                 return -rte_errno;
4091                         flow->rix_mreg_copy = mcp_res->idx;
4092                         if (dev->data->dev_started) {
4093                                 mcp_res->appcnt++;
4094                                 flow->copy_applied = 1;
4095                         }
4096                         return 0;
4097                 case RTE_FLOW_ACTION_TYPE_MARK:
4098                         mark = (const struct rte_flow_action_mark *)
4099                                 actions->conf;
4100                         mcp_res =
4101                                 flow_mreg_add_copy_action(dev, mark->id, error);
4102                         if (!mcp_res)
4103                                 return -rte_errno;
4104                         flow->rix_mreg_copy = mcp_res->idx;
4105                         if (dev->data->dev_started) {
4106                                 mcp_res->appcnt++;
4107                                 flow->copy_applied = 1;
4108                         }
4109                         return 0;
4110                 default:
4111                         break;
4112                 }
4113         }
4114         return 0;
4115 }
4116
4117 #define MLX5_MAX_SPLIT_ACTIONS 24
4118 #define MLX5_MAX_SPLIT_ITEMS 24
4119
4120 /**
4121  * Split the hairpin flow.
4122  * Since HW can't support encap and push-vlan on Rx, we move these
4123  * actions to Tx.
4124  * If the count action is after the encap then we also
4125  * move the count action. in this case the count will also measure
4126  * the outer bytes.
4127  *
4128  * @param dev
4129  *   Pointer to Ethernet device.
4130  * @param[in] actions
4131  *   Associated actions (list terminated by the END action).
4132  * @param[out] actions_rx
4133  *   Rx flow actions.
4134  * @param[out] actions_tx
4135  *   Tx flow actions..
4136  * @param[out] pattern_tx
4137  *   The pattern items for the Tx flow.
4138  * @param[out] flow_id
4139  *   The flow ID connected to this flow.
4140  *
4141  * @return
4142  *   0 on success.
4143  */
4144 static int
4145 flow_hairpin_split(struct rte_eth_dev *dev,
4146                    const struct rte_flow_action actions[],
4147                    struct rte_flow_action actions_rx[],
4148                    struct rte_flow_action actions_tx[],
4149                    struct rte_flow_item pattern_tx[],
4150                    uint32_t *flow_id)
4151 {
4152         struct mlx5_priv *priv = dev->data->dev_private;
4153         const struct rte_flow_action_raw_encap *raw_encap;
4154         const struct rte_flow_action_raw_decap *raw_decap;
4155         struct mlx5_rte_flow_action_set_tag *set_tag;
4156         struct rte_flow_action *tag_action;
4157         struct mlx5_rte_flow_item_tag *tag_item;
4158         struct rte_flow_item *item;
4159         char *addr;
4160         int encap = 0;
4161
4162         mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
4163         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4164                 switch (actions->type) {
4165                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4166                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4167                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4168                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4169                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4170                         rte_memcpy(actions_tx, actions,
4171                                sizeof(struct rte_flow_action));
4172                         actions_tx++;
4173                         break;
4174                 case RTE_FLOW_ACTION_TYPE_COUNT:
4175                         if (encap) {
4176                                 rte_memcpy(actions_tx, actions,
4177                                            sizeof(struct rte_flow_action));
4178                                 actions_tx++;
4179                         } else {
4180                                 rte_memcpy(actions_rx, actions,
4181                                            sizeof(struct rte_flow_action));
4182                                 actions_rx++;
4183                         }
4184                         break;
4185                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4186                         raw_encap = actions->conf;
4187                         if (raw_encap->size >
4188                             (sizeof(struct rte_flow_item_eth) +
4189                              sizeof(struct rte_flow_item_ipv4))) {
4190                                 memcpy(actions_tx, actions,
4191                                        sizeof(struct rte_flow_action));
4192                                 actions_tx++;
4193                                 encap = 1;
4194                         } else {
4195                                 rte_memcpy(actions_rx, actions,
4196                                            sizeof(struct rte_flow_action));
4197                                 actions_rx++;
4198                         }
4199                         break;
4200                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4201                         raw_decap = actions->conf;
4202                         if (raw_decap->size <
4203                             (sizeof(struct rte_flow_item_eth) +
4204                              sizeof(struct rte_flow_item_ipv4))) {
4205                                 memcpy(actions_tx, actions,
4206                                        sizeof(struct rte_flow_action));
4207                                 actions_tx++;
4208                         } else {
4209                                 rte_memcpy(actions_rx, actions,
4210                                            sizeof(struct rte_flow_action));
4211                                 actions_rx++;
4212                         }
4213                         break;
4214                 default:
4215                         rte_memcpy(actions_rx, actions,
4216                                    sizeof(struct rte_flow_action));
4217                         actions_rx++;
4218                         break;
4219                 }
4220         }
4221         /* Add set meta action and end action for the Rx flow. */
4222         tag_action = actions_rx;
4223         tag_action->type = (enum rte_flow_action_type)
4224                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4225         actions_rx++;
4226         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4227         actions_rx++;
4228         set_tag = (void *)actions_rx;
4229         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4230         MLX5_ASSERT(set_tag->id > REG_NON);
4231         set_tag->data = *flow_id;
4232         tag_action->conf = set_tag;
4233         /* Create Tx item list. */
4234         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4235         addr = (void *)&pattern_tx[2];
4236         item = pattern_tx;
4237         item->type = (enum rte_flow_item_type)
4238                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4239         tag_item = (void *)addr;
4240         tag_item->data = *flow_id;
4241         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4242         MLX5_ASSERT(set_tag->id > REG_NON);
4243         item->spec = tag_item;
4244         addr += sizeof(struct mlx5_rte_flow_item_tag);
4245         tag_item = (void *)addr;
4246         tag_item->data = UINT32_MAX;
4247         tag_item->id = UINT16_MAX;
4248         item->mask = tag_item;
4249         item->last = NULL;
4250         item++;
4251         item->type = RTE_FLOW_ITEM_TYPE_END;
4252         return 0;
4253 }
4254
4255 __extension__
4256 union tunnel_offload_mark {
4257         uint32_t val;
4258         struct {
4259                 uint32_t app_reserve:8;
4260                 uint32_t table_id:15;
4261                 uint32_t transfer:1;
4262                 uint32_t _unused_:8;
4263         };
4264 };
4265
4266 struct tunnel_default_miss_ctx {
4267         uint16_t *queue;
4268         __extension__
4269         union {
4270                 struct rte_flow_action_rss action_rss;
4271                 struct rte_flow_action_queue miss_queue;
4272                 struct rte_flow_action_jump miss_jump;
4273                 uint8_t raw[0];
4274         };
4275 };
4276
4277 static int
4278 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
4279                              struct rte_flow *flow,
4280                              const struct rte_flow_attr *attr,
4281                              const struct rte_flow_action *app_actions,
4282                              uint32_t flow_idx,
4283                              struct tunnel_default_miss_ctx *ctx,
4284                              struct rte_flow_error *error)
4285 {
4286         struct mlx5_priv *priv = dev->data->dev_private;
4287         struct mlx5_flow *dev_flow;
4288         struct rte_flow_attr miss_attr = *attr;
4289         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
4290         const struct rte_flow_item miss_items[2] = {
4291                 {
4292                         .type = RTE_FLOW_ITEM_TYPE_ETH,
4293                         .spec = NULL,
4294                         .last = NULL,
4295                         .mask = NULL
4296                 },
4297                 {
4298                         .type = RTE_FLOW_ITEM_TYPE_END,
4299                         .spec = NULL,
4300                         .last = NULL,
4301                         .mask = NULL
4302                 }
4303         };
4304         union tunnel_offload_mark mark_id;
4305         struct rte_flow_action_mark miss_mark;
4306         struct rte_flow_action miss_actions[3] = {
4307                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
4308                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
4309         };
4310         const struct rte_flow_action_jump *jump_data;
4311         uint32_t i, flow_table = 0; /* prevent compilation warning */
4312         struct flow_grp_info grp_info = {
4313                 .external = 1,
4314                 .transfer = attr->transfer,
4315                 .fdb_def_rule = !!priv->fdb_def_rule,
4316                 .std_tbl_fix = 0,
4317         };
4318         int ret;
4319
4320         if (!attr->transfer) {
4321                 uint32_t q_size;
4322
4323                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
4324                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
4325                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
4326                                          0, SOCKET_ID_ANY);
4327                 if (!ctx->queue)
4328                         return rte_flow_error_set
4329                                 (error, ENOMEM,
4330                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4331                                 NULL, "invalid default miss RSS");
4332                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
4333                 ctx->action_rss.level = 0,
4334                 ctx->action_rss.types = priv->rss_conf.rss_hf,
4335                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
4336                 ctx->action_rss.queue_num = priv->reta_idx_n,
4337                 ctx->action_rss.key = priv->rss_conf.rss_key,
4338                 ctx->action_rss.queue = ctx->queue;
4339                 if (!priv->reta_idx_n || !priv->rxqs_n)
4340                         return rte_flow_error_set
4341                                 (error, EINVAL,
4342                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4343                                 NULL, "invalid port configuration");
4344                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
4345                         ctx->action_rss.types = 0;
4346                 for (i = 0; i != priv->reta_idx_n; ++i)
4347                         ctx->queue[i] = (*priv->reta_idx)[i];
4348         } else {
4349                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
4350                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
4351         }
4352         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
4353         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
4354         jump_data = app_actions->conf;
4355         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
4356         miss_attr.group = jump_data->group;
4357         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
4358                                        &flow_table, grp_info, error);
4359         if (ret)
4360                 return rte_flow_error_set(error, EINVAL,
4361                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4362                                           NULL, "invalid tunnel id");
4363         mark_id.app_reserve = 0;
4364         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
4365         mark_id.transfer = !!attr->transfer;
4366         mark_id._unused_ = 0;
4367         miss_mark.id = mark_id.val;
4368         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
4369                                     miss_items, miss_actions, flow_idx, error);
4370         if (!dev_flow)
4371                 return -rte_errno;
4372         dev_flow->flow = flow;
4373         dev_flow->external = true;
4374         dev_flow->tunnel = tunnel;
4375         /* Subflow object was created, we must include one in the list. */
4376         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4377                       dev_flow->handle, next);
4378         DRV_LOG(DEBUG,
4379                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
4380                 dev->data->port_id, tunnel->app_tunnel.type,
4381                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
4382         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
4383                                   miss_actions, error);
4384         if (!ret)
4385                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
4386                                                   error);
4387
4388         return ret;
4389 }
4390
4391 /**
4392  * The last stage of splitting chain, just creates the subflow
4393  * without any modification.
4394  *
4395  * @param[in] dev
4396  *   Pointer to Ethernet device.
4397  * @param[in] flow
4398  *   Parent flow structure pointer.
4399  * @param[in, out] sub_flow
4400  *   Pointer to return the created subflow, may be NULL.
4401  * @param[in] prefix_layers
4402  *   Prefix subflow layers, may be 0.
4403  * @param[in] prefix_mark
4404  *   Prefix subflow mark flag, may be 0.
4405  * @param[in] attr
4406  *   Flow rule attributes.
4407  * @param[in] items
4408  *   Pattern specification (list terminated by the END pattern item).
4409  * @param[in] actions
4410  *   Associated actions (list terminated by the END action).
4411  * @param[in] external
4412  *   This flow rule is created by request external to PMD.
4413  * @param[in] flow_idx
4414  *   This memory pool index to the flow.
4415  * @param[out] error
4416  *   Perform verbose error reporting if not NULL.
4417  * @return
4418  *   0 on success, negative value otherwise
4419  */
4420 static int
4421 flow_create_split_inner(struct rte_eth_dev *dev,
4422                         struct rte_flow *flow,
4423                         struct mlx5_flow **sub_flow,
4424                         uint64_t prefix_layers,
4425                         uint32_t prefix_mark,
4426                         const struct rte_flow_attr *attr,
4427                         const struct rte_flow_item items[],
4428                         const struct rte_flow_action actions[],
4429                         bool external, uint32_t flow_idx,
4430                         struct rte_flow_error *error)
4431 {
4432         struct mlx5_flow *dev_flow;
4433
4434         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4435                 flow_idx, error);
4436         if (!dev_flow)
4437                 return -rte_errno;
4438         dev_flow->flow = flow;
4439         dev_flow->external = external;
4440         /* Subflow object was created, we must include one in the list. */
4441         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4442                       dev_flow->handle, next);
4443         /*
4444          * If dev_flow is as one of the suffix flow, some actions in suffix
4445          * flow may need some user defined item layer flags, and pass the
4446          * Metadate rxq mark flag to suffix flow as well.
4447          */
4448         if (prefix_layers)
4449                 dev_flow->handle->layers = prefix_layers;
4450         if (prefix_mark)
4451                 dev_flow->handle->mark = 1;
4452         if (sub_flow)
4453                 *sub_flow = dev_flow;
4454         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4455 }
4456
4457 /**
4458  * Split the meter flow.
4459  *
4460  * As meter flow will split to three sub flow, other than meter
4461  * action, the other actions make sense to only meter accepts
4462  * the packet. If it need to be dropped, no other additional
4463  * actions should be take.
4464  *
4465  * One kind of special action which decapsulates the L3 tunnel
4466  * header will be in the prefix sub flow, as not to take the
4467  * L3 tunnel header into account.
4468  *
4469  * @param dev
4470  *   Pointer to Ethernet device.
4471  * @param[in] items
4472  *   Pattern specification (list terminated by the END pattern item).
4473  * @param[out] sfx_items
4474  *   Suffix flow match items (list terminated by the END pattern item).
4475  * @param[in] actions
4476  *   Associated actions (list terminated by the END action).
4477  * @param[out] actions_sfx
4478  *   Suffix flow actions.
4479  * @param[out] actions_pre
4480  *   Prefix flow actions.
4481  * @param[out] pattern_sfx
4482  *   The pattern items for the suffix flow.
4483  * @param[out] tag_sfx
4484  *   Pointer to suffix flow tag.
4485  *
4486  * @return
4487  *   0 on success.
4488  */
4489 static int
4490 flow_meter_split_prep(struct rte_eth_dev *dev,
4491                  const struct rte_flow_item items[],
4492                  struct rte_flow_item sfx_items[],
4493                  const struct rte_flow_action actions[],
4494                  struct rte_flow_action actions_sfx[],
4495                  struct rte_flow_action actions_pre[])
4496 {
4497         struct rte_flow_action *tag_action = NULL;
4498         struct rte_flow_item *tag_item;
4499         struct mlx5_rte_flow_action_set_tag *set_tag;
4500         struct rte_flow_error error;
4501         const struct rte_flow_action_raw_encap *raw_encap;
4502         const struct rte_flow_action_raw_decap *raw_decap;
4503         struct mlx5_rte_flow_item_tag *tag_spec;
4504         struct mlx5_rte_flow_item_tag *tag_mask;
4505         uint32_t tag_id;
4506         bool copy_vlan = false;
4507
4508         /* Prepare the actions for prefix and suffix flow. */
4509         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4510                 struct rte_flow_action **action_cur = NULL;
4511
4512                 switch (actions->type) {
4513                 case RTE_FLOW_ACTION_TYPE_METER:
4514                         /* Add the extra tag action first. */
4515                         tag_action = actions_pre;
4516                         tag_action->type = (enum rte_flow_action_type)
4517                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4518                         actions_pre++;
4519                         action_cur = &actions_pre;
4520                         break;
4521                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4522                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4523                         action_cur = &actions_pre;
4524                         break;
4525                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4526                         raw_encap = actions->conf;
4527                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4528                                 action_cur = &actions_pre;
4529                         break;
4530                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4531                         raw_decap = actions->conf;
4532                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4533                                 action_cur = &actions_pre;
4534                         break;
4535                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4536                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4537                         copy_vlan = true;
4538                         break;
4539                 default:
4540                         break;
4541                 }
4542                 if (!action_cur)
4543                         action_cur = &actions_sfx;
4544                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4545                 (*action_cur)++;
4546         }
4547         /* Add end action to the actions. */
4548         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4549         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4550         actions_pre++;
4551         /* Set the tag. */
4552         set_tag = (void *)actions_pre;
4553         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4554         /*
4555          * Get the id from the qrss_pool to make qrss share the id with meter.
4556          */
4557         tag_id = flow_qrss_get_id(dev);
4558         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4559         assert(tag_action);
4560         tag_action->conf = set_tag;
4561         /* Prepare the suffix subflow items. */
4562         tag_item = sfx_items++;
4563         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4564                 int item_type = items->type;
4565
4566                 switch (item_type) {
4567                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4568                         memcpy(sfx_items, items, sizeof(*sfx_items));
4569                         sfx_items++;
4570                         break;
4571                 case RTE_FLOW_ITEM_TYPE_VLAN:
4572                         if (copy_vlan) {
4573                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4574                                 /*
4575                                  * Convert to internal match item, it is used
4576                                  * for vlan push and set vid.
4577                                  */
4578                                 sfx_items->type = (enum rte_flow_item_type)
4579                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4580                                 sfx_items++;
4581                         }
4582                         break;
4583                 default:
4584                         break;
4585                 }
4586         }
4587         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4588         sfx_items++;
4589         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4590         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4591         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4592         tag_mask = tag_spec + 1;
4593         tag_mask->data = 0xffffff00;
4594         tag_item->type = (enum rte_flow_item_type)
4595                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4596         tag_item->spec = tag_spec;
4597         tag_item->last = NULL;
4598         tag_item->mask = tag_mask;
4599         return tag_id;
4600 }
4601
4602 /**
4603  * Split action list having QUEUE/RSS for metadata register copy.
4604  *
4605  * Once Q/RSS action is detected in user's action list, the flow action
4606  * should be split in order to copy metadata registers, which will happen in
4607  * RX_CP_TBL like,
4608  *   - CQE->flow_tag := reg_c[1] (MARK)
4609  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4610  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4611  * This is because the last action of each flow must be a terminal action
4612  * (QUEUE, RSS or DROP).
4613  *
4614  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4615  * stored and kept in the mlx5_flow structure per each sub_flow.
4616  *
4617  * The Q/RSS action is replaced with,
4618  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4619  * And the following JUMP action is added at the end,
4620  *   - JUMP, to RX_CP_TBL.
4621  *
4622  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4623  * flow_create_split_metadata() routine. The flow will look like,
4624  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4625  *
4626  * @param dev
4627  *   Pointer to Ethernet device.
4628  * @param[out] split_actions
4629  *   Pointer to store split actions to jump to CP_TBL.
4630  * @param[in] actions
4631  *   Pointer to the list of original flow actions.
4632  * @param[in] qrss
4633  *   Pointer to the Q/RSS action.
4634  * @param[in] actions_n
4635  *   Number of original actions.
4636  * @param[out] error
4637  *   Perform verbose error reporting if not NULL.
4638  *
4639  * @return
4640  *   non-zero unique flow_id on success, otherwise 0 and
4641  *   error/rte_error are set.
4642  */
4643 static uint32_t
4644 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4645                           struct rte_flow_action *split_actions,
4646                           const struct rte_flow_action *actions,
4647                           const struct rte_flow_action *qrss,
4648                           int actions_n, struct rte_flow_error *error)
4649 {
4650         struct mlx5_rte_flow_action_set_tag *set_tag;
4651         struct rte_flow_action_jump *jump;
4652         const int qrss_idx = qrss - actions;
4653         uint32_t flow_id = 0;
4654         int ret = 0;
4655
4656         /*
4657          * Given actions will be split
4658          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4659          * - Add jump to mreg CP_TBL.
4660          * As a result, there will be one more action.
4661          */
4662         ++actions_n;
4663         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4664         set_tag = (void *)(split_actions + actions_n);
4665         /*
4666          * If tag action is not set to void(it means we are not the meter
4667          * suffix flow), add the tag action. Since meter suffix flow already
4668          * has the tag added.
4669          */
4670         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4671                 /*
4672                  * Allocate the new subflow ID. This one is unique within
4673                  * device and not shared with representors. Otherwise,
4674                  * we would have to resolve multi-thread access synch
4675                  * issue. Each flow on the shared device is appended
4676                  * with source vport identifier, so the resulting
4677                  * flows will be unique in the shared (by master and
4678                  * representors) domain even if they have coinciding
4679                  * IDs.
4680                  */
4681                 flow_id = flow_qrss_get_id(dev);
4682                 if (!flow_id)
4683                         return rte_flow_error_set(error, ENOMEM,
4684                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4685                                                   NULL, "can't allocate id "
4686                                                   "for split Q/RSS subflow");
4687                 /* Internal SET_TAG action to set flow ID. */
4688                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4689                         .data = flow_id,
4690                 };
4691                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4692                 if (ret < 0)
4693                         return ret;
4694                 set_tag->id = ret;
4695                 /* Construct new actions array. */
4696                 /* Replace QUEUE/RSS action. */
4697                 split_actions[qrss_idx] = (struct rte_flow_action){
4698                         .type = (enum rte_flow_action_type)
4699                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4700                         .conf = set_tag,
4701                 };
4702         }
4703         /* JUMP action to jump to mreg copy table (CP_TBL). */
4704         jump = (void *)(set_tag + 1);
4705         *jump = (struct rte_flow_action_jump){
4706                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4707         };
4708         split_actions[actions_n - 2] = (struct rte_flow_action){
4709                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4710                 .conf = jump,
4711         };
4712         split_actions[actions_n - 1] = (struct rte_flow_action){
4713                 .type = RTE_FLOW_ACTION_TYPE_END,
4714         };
4715         return flow_id;
4716 }
4717
4718 /**
4719  * Extend the given action list for Tx metadata copy.
4720  *
4721  * Copy the given action list to the ext_actions and add flow metadata register
4722  * copy action in order to copy reg_a set by WQE to reg_c[0].
4723  *
4724  * @param[out] ext_actions
4725  *   Pointer to the extended action list.
4726  * @param[in] actions
4727  *   Pointer to the list of actions.
4728  * @param[in] actions_n
4729  *   Number of actions in the list.
4730  * @param[out] error
4731  *   Perform verbose error reporting if not NULL.
4732  * @param[in] encap_idx
4733  *   The encap action inndex.
4734  *
4735  * @return
4736  *   0 on success, negative value otherwise
4737  */
4738 static int
4739 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4740                        struct rte_flow_action *ext_actions,
4741                        const struct rte_flow_action *actions,
4742                        int actions_n, struct rte_flow_error *error,
4743                        int encap_idx)
4744 {
4745         struct mlx5_flow_action_copy_mreg *cp_mreg =
4746                 (struct mlx5_flow_action_copy_mreg *)
4747                         (ext_actions + actions_n + 1);
4748         int ret;
4749
4750         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4751         if (ret < 0)
4752                 return ret;
4753         cp_mreg->dst = ret;
4754         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4755         if (ret < 0)
4756                 return ret;
4757         cp_mreg->src = ret;
4758         if (encap_idx != 0)
4759                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4760         if (encap_idx == actions_n - 1) {
4761                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4762                         .type = (enum rte_flow_action_type)
4763                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4764                         .conf = cp_mreg,
4765                 };
4766                 ext_actions[actions_n] = (struct rte_flow_action){
4767                         .type = RTE_FLOW_ACTION_TYPE_END,
4768                 };
4769         } else {
4770                 ext_actions[encap_idx] = (struct rte_flow_action){
4771                         .type = (enum rte_flow_action_type)
4772                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4773                         .conf = cp_mreg,
4774                 };
4775                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4776                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4777         }
4778         return 0;
4779 }
4780
4781 /**
4782  * Check the match action from the action list.
4783  *
4784  * @param[in] actions
4785  *   Pointer to the list of actions.
4786  * @param[in] attr
4787  *   Flow rule attributes.
4788  * @param[in] action
4789  *   The action to be check if exist.
4790  * @param[out] match_action_pos
4791  *   Pointer to the position of the matched action if exists, otherwise is -1.
4792  * @param[out] qrss_action_pos
4793  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4794  *
4795  * @return
4796  *   > 0 the total number of actions.
4797  *   0 if not found match action in action list.
4798  */
4799 static int
4800 flow_check_match_action(const struct rte_flow_action actions[],
4801                         const struct rte_flow_attr *attr,
4802                         enum rte_flow_action_type action,
4803                         int *match_action_pos, int *qrss_action_pos)
4804 {
4805         const struct rte_flow_action_sample *sample;
4806         int actions_n = 0;
4807         int jump_flag = 0;
4808         uint32_t ratio = 0;
4809         int sub_type = 0;
4810         int flag = 0;
4811
4812         *match_action_pos = -1;
4813         *qrss_action_pos = -1;
4814         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4815                 if (actions->type == action) {
4816                         flag = 1;
4817                         *match_action_pos = actions_n;
4818                 }
4819                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4820                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4821                         *qrss_action_pos = actions_n;
4822                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4823                         jump_flag = 1;
4824                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4825                         sample = actions->conf;
4826                         ratio = sample->ratio;
4827                         sub_type = ((const struct rte_flow_action *)
4828                                         (sample->actions))->type;
4829                 }
4830                 actions_n++;
4831         }
4832         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4833                 if (ratio == 1) {
4834                         /* JUMP Action not support for Mirroring;
4835                          * Mirroring support multi-destination;
4836                          */
4837                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4838                                 flag = 0;
4839                 }
4840         }
4841         /* Count RTE_FLOW_ACTION_TYPE_END. */
4842         return flag ? actions_n + 1 : 0;
4843 }
4844
4845 #define SAMPLE_SUFFIX_ITEM 2
4846
4847 /**
4848  * Split the sample flow.
4849  *
4850  * As sample flow will split to two sub flow, sample flow with
4851  * sample action, the other actions will move to new suffix flow.
4852  *
4853  * Also add unique tag id with tag action in the sample flow,
4854  * the same tag id will be as match in the suffix flow.
4855  *
4856  * @param dev
4857  *   Pointer to Ethernet device.
4858  * @param[in] fdb_tx
4859  *   FDB egress flow flag.
4860  * @param[out] sfx_items
4861  *   Suffix flow match items (list terminated by the END pattern item).
4862  * @param[in] actions
4863  *   Associated actions (list terminated by the END action).
4864  * @param[out] actions_sfx
4865  *   Suffix flow actions.
4866  * @param[out] actions_pre
4867  *   Prefix flow actions.
4868  * @param[in] actions_n
4869  *  The total number of actions.
4870  * @param[in] sample_action_pos
4871  *   The sample action position.
4872  * @param[in] qrss_action_pos
4873  *   The Queue/RSS action position.
4874  * @param[out] error
4875  *   Perform verbose error reporting if not NULL.
4876  *
4877  * @return
4878  *   0 on success, or unique flow_id, a negative errno value
4879  *   otherwise and rte_errno is set.
4880  */
4881 static int
4882 flow_sample_split_prep(struct rte_eth_dev *dev,
4883                        uint32_t fdb_tx,
4884                        struct rte_flow_item sfx_items[],
4885                        const struct rte_flow_action actions[],
4886                        struct rte_flow_action actions_sfx[],
4887                        struct rte_flow_action actions_pre[],
4888                        int actions_n,
4889                        int sample_action_pos,
4890                        int qrss_action_pos,
4891                        struct rte_flow_error *error)
4892 {
4893         struct mlx5_rte_flow_action_set_tag *set_tag;
4894         struct mlx5_rte_flow_item_tag *tag_spec;
4895         struct mlx5_rte_flow_item_tag *tag_mask;
4896         uint32_t tag_id = 0;
4897         int index;
4898         int ret;
4899
4900         if (sample_action_pos < 0)
4901                 return rte_flow_error_set(error, EINVAL,
4902                                           RTE_FLOW_ERROR_TYPE_ACTION,
4903                                           NULL, "invalid position of sample "
4904                                           "action in list");
4905         if (!fdb_tx) {
4906                 /* Prepare the prefix tag action. */
4907                 set_tag = (void *)(actions_pre + actions_n + 1);
4908                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4909                 if (ret < 0)
4910                         return ret;
4911                 set_tag->id = ret;
4912                 tag_id = flow_qrss_get_id(dev);
4913                 set_tag->data = tag_id;
4914                 /* Prepare the suffix subflow items. */
4915                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4916                 tag_spec->data = tag_id;
4917                 tag_spec->id = set_tag->id;
4918                 tag_mask = tag_spec + 1;
4919                 tag_mask->data = UINT32_MAX;
4920                 sfx_items[0] = (struct rte_flow_item){
4921                         .type = (enum rte_flow_item_type)
4922                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4923                         .spec = tag_spec,
4924                         .last = NULL,
4925                         .mask = tag_mask,
4926                 };
4927                 sfx_items[1] = (struct rte_flow_item){
4928                         .type = (enum rte_flow_item_type)
4929                                 RTE_FLOW_ITEM_TYPE_END,
4930                 };
4931         }
4932         /* Prepare the actions for prefix and suffix flow. */
4933         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4934                 index = qrss_action_pos;
4935                 /* Put the preceding the Queue/RSS action into prefix flow. */
4936                 if (index != 0)
4937                         memcpy(actions_pre, actions,
4938                                sizeof(struct rte_flow_action) * index);
4939                 /* Put others preceding the sample action into prefix flow. */
4940                 if (sample_action_pos > index + 1)
4941                         memcpy(actions_pre + index, actions + index + 1,
4942                                sizeof(struct rte_flow_action) *
4943                                (sample_action_pos - index - 1));
4944                 index = sample_action_pos - 1;
4945                 /* Put Queue/RSS action into Suffix flow. */
4946                 memcpy(actions_sfx, actions + qrss_action_pos,
4947                        sizeof(struct rte_flow_action));
4948                 actions_sfx++;
4949         } else {
4950                 index = sample_action_pos;
4951                 if (index != 0)
4952                         memcpy(actions_pre, actions,
4953                                sizeof(struct rte_flow_action) * index);
4954         }
4955         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4956         if (!fdb_tx) {
4957                 actions_pre[index++] =
4958                         (struct rte_flow_action){
4959                         .type = (enum rte_flow_action_type)
4960                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4961                         .conf = set_tag,
4962                 };
4963         }
4964         memcpy(actions_pre + index, actions + sample_action_pos,
4965                sizeof(struct rte_flow_action));
4966         index += 1;
4967         actions_pre[index] = (struct rte_flow_action){
4968                 .type = (enum rte_flow_action_type)
4969                         RTE_FLOW_ACTION_TYPE_END,
4970         };
4971         /* Put the actions after sample into Suffix flow. */
4972         memcpy(actions_sfx, actions + sample_action_pos + 1,
4973                sizeof(struct rte_flow_action) *
4974                (actions_n - sample_action_pos - 1));
4975         return tag_id;
4976 }
4977
4978 /**
4979  * The splitting for metadata feature.
4980  *
4981  * - Q/RSS action on NIC Rx should be split in order to pass by
4982  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4983  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4984  *
4985  * - All the actions on NIC Tx should have a mreg copy action to
4986  *   copy reg_a from WQE to reg_c[0].
4987  *
4988  * @param dev
4989  *   Pointer to Ethernet device.
4990  * @param[in] flow
4991  *   Parent flow structure pointer.
4992  * @param[in] prefix_layers
4993  *   Prefix flow layer flags.
4994  * @param[in] prefix_mark
4995  *   Prefix subflow mark flag, may be 0.
4996  * @param[in] attr
4997  *   Flow rule attributes.
4998  * @param[in] items
4999  *   Pattern specification (list terminated by the END pattern item).
5000  * @param[in] actions
5001  *   Associated actions (list terminated by the END action).
5002  * @param[in] external
5003  *   This flow rule is created by request external to PMD.
5004  * @param[in] flow_idx
5005  *   This memory pool index to the flow.
5006  * @param[out] error
5007  *   Perform verbose error reporting if not NULL.
5008  * @return
5009  *   0 on success, negative value otherwise
5010  */
5011 static int
5012 flow_create_split_metadata(struct rte_eth_dev *dev,
5013                            struct rte_flow *flow,
5014                            uint64_t prefix_layers,
5015                            uint32_t prefix_mark,
5016                            const struct rte_flow_attr *attr,
5017                            const struct rte_flow_item items[],
5018                            const struct rte_flow_action actions[],
5019                            bool external, uint32_t flow_idx,
5020                            struct rte_flow_error *error)
5021 {
5022         struct mlx5_priv *priv = dev->data->dev_private;
5023         struct mlx5_dev_config *config = &priv->config;
5024         const struct rte_flow_action *qrss = NULL;
5025         struct rte_flow_action *ext_actions = NULL;
5026         struct mlx5_flow *dev_flow = NULL;
5027         uint32_t qrss_id = 0;
5028         int mtr_sfx = 0;
5029         size_t act_size;
5030         int actions_n;
5031         int encap_idx;
5032         int ret;
5033
5034         /* Check whether extensive metadata feature is engaged. */
5035         if (!config->dv_flow_en ||
5036             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5037             !mlx5_flow_ext_mreg_supported(dev))
5038                 return flow_create_split_inner(dev, flow, NULL, prefix_layers,
5039                                                prefix_mark, attr, items,
5040                                                actions, external, flow_idx,
5041                                                error);
5042         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5043                                                            &encap_idx);
5044         if (qrss) {
5045                 /* Exclude hairpin flows from splitting. */
5046                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5047                         const struct rte_flow_action_queue *queue;
5048
5049                         queue = qrss->conf;
5050                         if (mlx5_rxq_get_type(dev, queue->index) ==
5051                             MLX5_RXQ_TYPE_HAIRPIN)
5052                                 qrss = NULL;
5053                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5054                         const struct rte_flow_action_rss *rss;
5055
5056                         rss = qrss->conf;
5057                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5058                             MLX5_RXQ_TYPE_HAIRPIN)
5059                                 qrss = NULL;
5060                 }
5061         }
5062         if (qrss) {
5063                 /* Check if it is in meter suffix table. */
5064                 mtr_sfx = attr->group == (attr->transfer ?
5065                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
5066                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
5067                 /*
5068                  * Q/RSS action on NIC Rx should be split in order to pass by
5069                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5070                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5071                  */
5072                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5073                            sizeof(struct rte_flow_action_set_tag) +
5074                            sizeof(struct rte_flow_action_jump);
5075                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5076                                           SOCKET_ID_ANY);
5077                 if (!ext_actions)
5078                         return rte_flow_error_set(error, ENOMEM,
5079                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5080                                                   NULL, "no memory to split "
5081                                                   "metadata flow");
5082                 /*
5083                  * If we are the suffix flow of meter, tag already exist.
5084                  * Set the tag action to void.
5085                  */
5086                 if (mtr_sfx)
5087                         ext_actions[qrss - actions].type =
5088                                                 RTE_FLOW_ACTION_TYPE_VOID;
5089                 else
5090                         ext_actions[qrss - actions].type =
5091                                                 (enum rte_flow_action_type)
5092                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5093                 /*
5094                  * Create the new actions list with removed Q/RSS action
5095                  * and appended set tag and jump to register copy table
5096                  * (RX_CP_TBL). We should preallocate unique tag ID here
5097                  * in advance, because it is needed for set tag action.
5098                  */
5099                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5100                                                     qrss, actions_n, error);
5101                 if (!mtr_sfx && !qrss_id) {
5102                         ret = -rte_errno;
5103                         goto exit;
5104                 }
5105         } else if (attr->egress && !attr->transfer) {
5106                 /*
5107                  * All the actions on NIC Tx should have a metadata register
5108                  * copy action to copy reg_a from WQE to reg_c[meta]
5109                  */
5110                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5111                            sizeof(struct mlx5_flow_action_copy_mreg);
5112                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5113                                           SOCKET_ID_ANY);
5114                 if (!ext_actions)
5115                         return rte_flow_error_set(error, ENOMEM,
5116                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5117                                                   NULL, "no memory to split "
5118                                                   "metadata flow");
5119                 /* Create the action list appended with copy register. */
5120                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5121                                              actions_n, error, encap_idx);
5122                 if (ret < 0)
5123                         goto exit;
5124         }
5125         /* Add the unmodified original or prefix subflow. */
5126         ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers,
5127                                       prefix_mark, attr,
5128                                       items, ext_actions ? ext_actions :
5129                                       actions, external, flow_idx, error);
5130         if (ret < 0)
5131                 goto exit;
5132         MLX5_ASSERT(dev_flow);
5133         if (qrss) {
5134                 const struct rte_flow_attr q_attr = {
5135                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5136                         .ingress = 1,
5137                 };
5138                 /* Internal PMD action to set register. */
5139                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5140                         .data = qrss_id,
5141                         .id = REG_NON,
5142                 };
5143                 struct rte_flow_item q_items[] = {
5144                         {
5145                                 .type = (enum rte_flow_item_type)
5146                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5147                                 .spec = &q_tag_spec,
5148                                 .last = NULL,
5149                                 .mask = NULL,
5150                         },
5151                         {
5152                                 .type = RTE_FLOW_ITEM_TYPE_END,
5153                         },
5154                 };
5155                 struct rte_flow_action q_actions[] = {
5156                         {
5157                                 .type = qrss->type,
5158                                 .conf = qrss->conf,
5159                         },
5160                         {
5161                                 .type = RTE_FLOW_ACTION_TYPE_END,
5162                         },
5163                 };
5164                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5165
5166                 /*
5167                  * Configure the tag item only if there is no meter subflow.
5168                  * Since tag is already marked in the meter suffix subflow
5169                  * we can just use the meter suffix items as is.
5170                  */
5171                 if (qrss_id) {
5172                         /* Not meter subflow. */
5173                         MLX5_ASSERT(!mtr_sfx);
5174                         /*
5175                          * Put unique id in prefix flow due to it is destroyed
5176                          * after suffix flow and id will be freed after there
5177                          * is no actual flows with this id and identifier
5178                          * reallocation becomes possible (for example, for
5179                          * other flows in other threads).
5180                          */
5181                         dev_flow->handle->split_flow_id = qrss_id;
5182                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5183                                                    error);
5184                         if (ret < 0)
5185                                 goto exit;
5186                         q_tag_spec.id = ret;
5187                 }
5188                 dev_flow = NULL;
5189                 /* Add suffix subflow to execute Q/RSS. */
5190                 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0,
5191                                               &q_attr, mtr_sfx ? items :
5192                                               q_items, q_actions,
5193                                               external, flow_idx, error);
5194                 if (ret < 0)
5195                         goto exit;
5196                 /* qrss ID should be freed if failed. */
5197                 qrss_id = 0;
5198                 MLX5_ASSERT(dev_flow);
5199         }
5200
5201 exit:
5202         /*
5203          * We do not destroy the partially created sub_flows in case of error.
5204          * These ones are included into parent flow list and will be destroyed
5205          * by flow_drv_destroy.
5206          */
5207         flow_qrss_free_id(dev, qrss_id);
5208         mlx5_free(ext_actions);
5209         return ret;
5210 }
5211
5212 /**
5213  * The splitting for meter feature.
5214  *
5215  * - The meter flow will be split to two flows as prefix and
5216  *   suffix flow. The packets make sense only it pass the prefix
5217  *   meter action.
5218  *
5219  * - Reg_C_5 is used for the packet to match betweend prefix and
5220  *   suffix flow.
5221  *
5222  * @param dev
5223  *   Pointer to Ethernet device.
5224  * @param[in] flow
5225  *   Parent flow structure pointer.
5226  * @param[in] prefix_layers
5227  *   Prefix subflow layers, may be 0.
5228  * @param[in] prefix_mark
5229  *   Prefix subflow mark flag, may be 0.
5230  * @param[in] attr
5231  *   Flow rule attributes.
5232  * @param[in] items
5233  *   Pattern specification (list terminated by the END pattern item).
5234  * @param[in] actions
5235  *   Associated actions (list terminated by the END action).
5236  * @param[in] external
5237  *   This flow rule is created by request external to PMD.
5238  * @param[in] flow_idx
5239  *   This memory pool index to the flow.
5240  * @param[out] error
5241  *   Perform verbose error reporting if not NULL.
5242  * @return
5243  *   0 on success, negative value otherwise
5244  */
5245 static int
5246 flow_create_split_meter(struct rte_eth_dev *dev,
5247                         struct rte_flow *flow,
5248                         uint64_t prefix_layers,
5249                         uint32_t prefix_mark,
5250                         const struct rte_flow_attr *attr,
5251                         const struct rte_flow_item items[],
5252                         const struct rte_flow_action actions[],
5253                         bool external, uint32_t flow_idx,
5254                         struct rte_flow_error *error)
5255 {
5256         struct mlx5_priv *priv = dev->data->dev_private;
5257         struct rte_flow_action *sfx_actions = NULL;
5258         struct rte_flow_action *pre_actions = NULL;
5259         struct rte_flow_item *sfx_items = NULL;
5260         struct mlx5_flow *dev_flow = NULL;
5261         struct rte_flow_attr sfx_attr = *attr;
5262         uint32_t mtr = 0;
5263         uint32_t mtr_tag_id = 0;
5264         size_t act_size;
5265         size_t item_size;
5266         int actions_n = 0;
5267         int ret;
5268
5269         if (priv->mtr_en)
5270                 actions_n = flow_check_meter_action(actions, &mtr);
5271         if (mtr) {
5272                 /* The five prefix actions: meter, decap, encap, tag, end. */
5273                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
5274                            sizeof(struct mlx5_rte_flow_action_set_tag);
5275                 /* tag, vlan, port id, end. */
5276 #define METER_SUFFIX_ITEM 4
5277                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5278                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5279                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5280                                           0, SOCKET_ID_ANY);
5281                 if (!sfx_actions)
5282                         return rte_flow_error_set(error, ENOMEM,
5283                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5284                                                   NULL, "no memory to split "
5285                                                   "meter flow");
5286                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5287                              act_size);
5288                 pre_actions = sfx_actions + actions_n;
5289                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
5290                                                    actions, sfx_actions,
5291                                                    pre_actions);
5292                 if (!mtr_tag_id) {
5293                         ret = -rte_errno;
5294                         goto exit;
5295                 }
5296                 /* Add the prefix subflow. */
5297                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5298                                               prefix_layers, 0,
5299                                               attr, items,
5300                                               pre_actions, external,
5301                                               flow_idx, error);
5302                 if (ret) {
5303                         ret = -rte_errno;
5304                         goto exit;
5305                 }
5306                 dev_flow->handle->split_flow_id = mtr_tag_id;
5307                 /* Setting the sfx group atrr. */
5308                 sfx_attr.group = sfx_attr.transfer ?
5309                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
5310                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
5311         }
5312         /* Add the prefix subflow. */
5313         ret = flow_create_split_metadata(dev, flow, dev_flow ?
5314                                          flow_get_prefix_layer_flags(dev_flow) :
5315                                          prefix_layers, dev_flow ?
5316                                          dev_flow->handle->mark : prefix_mark,
5317                                          &sfx_attr, sfx_items ?
5318                                          sfx_items : items,
5319                                          sfx_actions ? sfx_actions : actions,
5320                                          external, flow_idx, error);
5321 exit:
5322         if (sfx_actions)
5323                 mlx5_free(sfx_actions);
5324         return ret;
5325 }
5326
5327 /**
5328  * The splitting for sample feature.
5329  *
5330  * Once Sample action is detected in the action list, the flow actions should
5331  * be split into prefix sub flow and suffix sub flow.
5332  *
5333  * The original items remain in the prefix sub flow, all actions preceding the
5334  * sample action and the sample action itself will be copied to the prefix
5335  * sub flow, the actions following the sample action will be copied to the
5336  * suffix sub flow, Queue action always be located in the suffix sub flow.
5337  *
5338  * In order to make the packet from prefix sub flow matches with suffix sub
5339  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5340  * flow uses tag item with the unique flow id.
5341  *
5342  * @param dev
5343  *   Pointer to Ethernet device.
5344  * @param[in] flow
5345  *   Parent flow structure pointer.
5346  * @param[in] attr
5347  *   Flow rule attributes.
5348  * @param[in] items
5349  *   Pattern specification (list terminated by the END pattern item).
5350  * @param[in] actions
5351  *   Associated actions (list terminated by the END action).
5352  * @param[in] external
5353  *   This flow rule is created by request external to PMD.
5354  * @param[in] flow_idx
5355  *   This memory pool index to the flow.
5356  * @param[out] error
5357  *   Perform verbose error reporting if not NULL.
5358  * @return
5359  *   0 on success, negative value otherwise
5360  */
5361 static int
5362 flow_create_split_sample(struct rte_eth_dev *dev,
5363                          struct rte_flow *flow,
5364                          const struct rte_flow_attr *attr,
5365                          const struct rte_flow_item items[],
5366                          const struct rte_flow_action actions[],
5367                          bool external, uint32_t flow_idx,
5368                          struct rte_flow_error *error)
5369 {
5370         struct mlx5_priv *priv = dev->data->dev_private;
5371         struct rte_flow_action *sfx_actions = NULL;
5372         struct rte_flow_action *pre_actions = NULL;
5373         struct rte_flow_item *sfx_items = NULL;
5374         struct mlx5_flow *dev_flow = NULL;
5375         struct rte_flow_attr sfx_attr = *attr;
5376 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5377         struct mlx5_flow_dv_sample_resource *sample_res;
5378         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5379         struct mlx5_flow_tbl_resource *sfx_tbl;
5380         union mlx5_flow_tbl_key sfx_table_key;
5381 #endif
5382         size_t act_size;
5383         size_t item_size;
5384         uint32_t fdb_tx = 0;
5385         int32_t tag_id = 0;
5386         int actions_n = 0;
5387         int sample_action_pos;
5388         int qrss_action_pos;
5389         int ret = 0;
5390
5391         if (priv->sampler_en)
5392                 actions_n = flow_check_match_action(actions, attr,
5393                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5394                                         &sample_action_pos, &qrss_action_pos);
5395         if (actions_n) {
5396                 /* The prefix actions must includes sample, tag, end. */
5397                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5398                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5399                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5400                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5401                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5402                                           item_size), 0, SOCKET_ID_ANY);
5403                 if (!sfx_actions)
5404                         return rte_flow_error_set(error, ENOMEM,
5405                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5406                                                   NULL, "no memory to split "
5407                                                   "sample flow");
5408                 /* The representor_id is -1 for uplink. */
5409                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5410                 if (!fdb_tx)
5411                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5412                                         + act_size);
5413                 pre_actions = sfx_actions + actions_n;
5414                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5415                                                 actions, sfx_actions,
5416                                                 pre_actions, actions_n,
5417                                                 sample_action_pos,
5418                                                 qrss_action_pos, error);
5419                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5420                         ret = -rte_errno;
5421                         goto exit;
5422                 }
5423                 /* Add the prefix subflow. */
5424                 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr,
5425                                               items, pre_actions, external,
5426                                               flow_idx, error);
5427                 if (ret) {
5428                         ret = -rte_errno;
5429                         goto exit;
5430                 }
5431                 dev_flow->handle->split_flow_id = tag_id;
5432 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5433                 /* Set the sfx group attr. */
5434                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5435                                         dev_flow->dv.sample_res;
5436                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5437                                         sample_res->normal_path_tbl;
5438                 sfx_tbl_data = container_of(sfx_tbl,
5439                                         struct mlx5_flow_tbl_data_entry, tbl);
5440                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5441                 sfx_attr.group = sfx_attr.transfer ?
5442                                         (sfx_table_key.table_id - 1) :
5443                                          sfx_table_key.table_id;
5444 #endif
5445         }
5446         /* Add the suffix subflow. */
5447         ret = flow_create_split_meter(dev, flow, dev_flow ?
5448                                  flow_get_prefix_layer_flags(dev_flow) : 0,
5449                                  dev_flow ? dev_flow->handle->mark : 0,
5450                                  &sfx_attr, sfx_items ? sfx_items : items,
5451                                  sfx_actions ? sfx_actions : actions,
5452                                  external, flow_idx, error);
5453 exit:
5454         if (sfx_actions)
5455                 mlx5_free(sfx_actions);
5456         return ret;
5457 }
5458
5459 /**
5460  * Split the flow to subflow set. The splitters might be linked
5461  * in the chain, like this:
5462  * flow_create_split_outer() calls:
5463  *   flow_create_split_meter() calls:
5464  *     flow_create_split_metadata(meter_subflow_0) calls:
5465  *       flow_create_split_inner(metadata_subflow_0)
5466  *       flow_create_split_inner(metadata_subflow_1)
5467  *       flow_create_split_inner(metadata_subflow_2)
5468  *     flow_create_split_metadata(meter_subflow_1) calls:
5469  *       flow_create_split_inner(metadata_subflow_0)
5470  *       flow_create_split_inner(metadata_subflow_1)
5471  *       flow_create_split_inner(metadata_subflow_2)
5472  *
5473  * This provide flexible way to add new levels of flow splitting.
5474  * The all of successfully created subflows are included to the
5475  * parent flow dev_flow list.
5476  *
5477  * @param dev
5478  *   Pointer to Ethernet device.
5479  * @param[in] flow
5480  *   Parent flow structure pointer.
5481  * @param[in] attr
5482  *   Flow rule attributes.
5483  * @param[in] items
5484  *   Pattern specification (list terminated by the END pattern item).
5485  * @param[in] actions
5486  *   Associated actions (list terminated by the END action).
5487  * @param[in] external
5488  *   This flow rule is created by request external to PMD.
5489  * @param[in] flow_idx
5490  *   This memory pool index to the flow.
5491  * @param[out] error
5492  *   Perform verbose error reporting if not NULL.
5493  * @return
5494  *   0 on success, negative value otherwise
5495  */
5496 static int
5497 flow_create_split_outer(struct rte_eth_dev *dev,
5498                         struct rte_flow *flow,
5499                         const struct rte_flow_attr *attr,
5500                         const struct rte_flow_item items[],
5501                         const struct rte_flow_action actions[],
5502                         bool external, uint32_t flow_idx,
5503                         struct rte_flow_error *error)
5504 {
5505         int ret;
5506
5507         ret = flow_create_split_sample(dev, flow, attr, items,
5508                                        actions, external, flow_idx, error);
5509         MLX5_ASSERT(ret <= 0);
5510         return ret;
5511 }
5512
5513 static struct mlx5_flow_tunnel *
5514 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5515                       const struct rte_flow_attr *attr,
5516                       const struct rte_flow_item items[],
5517                       const struct rte_flow_action actions[])
5518 {
5519         struct mlx5_flow_tunnel *tunnel;
5520
5521 #pragma GCC diagnostic push
5522 #pragma GCC diagnostic ignored "-Wcast-qual"
5523         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5524                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5525         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5526                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5527         else
5528                 tunnel = NULL;
5529 #pragma GCC diagnostic pop
5530
5531         return tunnel;
5532 }
5533
5534 /**
5535  * Create a flow and add it to @p list.
5536  *
5537  * @param dev
5538  *   Pointer to Ethernet device.
5539  * @param list
5540  *   Pointer to a TAILQ flow list. If this parameter NULL,
5541  *   no list insertion occurred, flow is just created,
5542  *   this is caller's responsibility to track the
5543  *   created flow.
5544  * @param[in] attr
5545  *   Flow rule attributes.
5546  * @param[in] items
5547  *   Pattern specification (list terminated by the END pattern item).
5548  * @param[in] actions
5549  *   Associated actions (list terminated by the END action).
5550  * @param[in] external
5551  *   This flow rule is created by request external to PMD.
5552  * @param[out] error
5553  *   Perform verbose error reporting if not NULL.
5554  *
5555  * @return
5556  *   A flow index on success, 0 otherwise and rte_errno is set.
5557  */
5558 static uint32_t
5559 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5560                  const struct rte_flow_attr *attr,
5561                  const struct rte_flow_item items[],
5562                  const struct rte_flow_action original_actions[],
5563                  bool external, struct rte_flow_error *error)
5564 {
5565         struct mlx5_priv *priv = dev->data->dev_private;
5566         struct rte_flow *flow = NULL;
5567         struct mlx5_flow *dev_flow;
5568         const struct rte_flow_action_rss *rss;
5569         struct mlx5_translated_shared_action
5570                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5571         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5572         union {
5573                 struct mlx5_flow_expand_rss buf;
5574                 uint8_t buffer[2048];
5575         } expand_buffer;
5576         union {
5577                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5578                 uint8_t buffer[2048];
5579         } actions_rx;
5580         union {
5581                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5582                 uint8_t buffer[2048];
5583         } actions_hairpin_tx;
5584         union {
5585                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5586                 uint8_t buffer[2048];
5587         } items_tx;
5588         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5589         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
5590                                               priv->rss_desc)[!!priv->flow_idx];
5591         const struct rte_flow_action *p_actions_rx;
5592         uint32_t i;
5593         uint32_t idx = 0;
5594         int hairpin_flow;
5595         uint32_t hairpin_id = 0;
5596         struct rte_flow_attr attr_tx = { .priority = 0 };
5597         struct rte_flow_attr attr_factor = {0};
5598         const struct rte_flow_action *actions;
5599         struct rte_flow_action *translated_actions = NULL;
5600         struct mlx5_flow_tunnel *tunnel;
5601         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5602         int ret = flow_shared_actions_translate(original_actions,
5603                                                 shared_actions,
5604                                                 &shared_actions_n,
5605                                                 &translated_actions, error);
5606
5607         if (ret < 0) {
5608                 MLX5_ASSERT(translated_actions == NULL);
5609                 return 0;
5610         }
5611         actions = translated_actions ? translated_actions : original_actions;
5612         memcpy((void *)&attr_factor, (const void *)attr, sizeof(*attr));
5613         p_actions_rx = actions;
5614         hairpin_flow = flow_check_hairpin_split(dev, &attr_factor, actions);
5615         ret = flow_drv_validate(dev, &attr_factor, items, p_actions_rx,
5616                                 external, hairpin_flow, error);
5617         if (ret < 0)
5618                 goto error_before_hairpin_split;
5619         if (hairpin_flow > 0) {
5620                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5621                         rte_errno = EINVAL;
5622                         goto error_before_hairpin_split;
5623                 }
5624                 flow_hairpin_split(dev, actions, actions_rx.actions,
5625                                    actions_hairpin_tx.actions, items_tx.items,
5626                                    &hairpin_id);
5627                 p_actions_rx = actions_rx.actions;
5628         }
5629         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5630         if (!flow) {
5631                 rte_errno = ENOMEM;
5632                 goto error_before_flow;
5633         }
5634         flow->drv_type = flow_get_drv_type(dev, &attr_factor);
5635         if (hairpin_id != 0)
5636                 flow->hairpin_flow_id = hairpin_id;
5637         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5638                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5639         memset(rss_desc, 0, sizeof(*rss_desc));
5640         rss = flow_get_rss_action(p_actions_rx);
5641         if (rss) {
5642                 /*
5643                  * The following information is required by
5644                  * mlx5_flow_hashfields_adjust() in advance.
5645                  */
5646                 rss_desc->level = rss->level;
5647                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5648                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5649         }
5650         flow->dev_handles = 0;
5651         if (rss && rss->types) {
5652                 unsigned int graph_root;
5653
5654                 graph_root = find_graph_root(items, rss->level);
5655                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5656                                            items, rss->types,
5657                                            mlx5_support_expansion, graph_root);
5658                 MLX5_ASSERT(ret > 0 &&
5659                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5660         } else {
5661                 buf->entries = 1;
5662                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5663         }
5664         flow->shared_rss = flow_get_shared_rss_action(shared_actions,
5665                                                       shared_actions_n);
5666         /*
5667          * Record the start index when there is a nested call. All sub-flows
5668          * need to be translated before another calling.
5669          * No need to use ping-pong buffer to save memory here.
5670          */
5671         if (priv->flow_idx) {
5672                 MLX5_ASSERT(!priv->flow_nested_idx);
5673                 priv->flow_nested_idx = priv->flow_idx;
5674         }
5675         for (i = 0; i < buf->entries; ++i) {
5676                 /*
5677                  * The splitter may create multiple dev_flows,
5678                  * depending on configuration. In the simplest
5679                  * case it just creates unmodified original flow.
5680                  */
5681                 ret = flow_create_split_outer(dev, flow, &attr_factor,
5682                                               buf->entry[i].pattern,
5683                                               p_actions_rx, external, idx,
5684                                               error);
5685                 if (ret < 0)
5686                         goto error;
5687                 if (is_flow_tunnel_steer_rule(dev, attr,
5688                                               buf->entry[i].pattern,
5689                                               p_actions_rx)) {
5690                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5691                                                            p_actions_rx,
5692                                                            idx,
5693                                                            &default_miss_ctx,
5694                                                            error);
5695                         if (ret < 0) {
5696                                 mlx5_free(default_miss_ctx.queue);
5697                                 goto error;
5698                         }
5699                 }
5700         }
5701         /* Create the tx flow. */
5702         if (hairpin_flow) {
5703                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5704                 attr_tx.ingress = 0;
5705                 attr_tx.egress = 1;
5706                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5707                                          actions_hairpin_tx.actions,
5708                                          idx, error);
5709                 if (!dev_flow)
5710                         goto error;
5711                 dev_flow->flow = flow;
5712                 dev_flow->external = 0;
5713                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5714                               dev_flow->handle, next);
5715                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5716                                          items_tx.items,
5717                                          actions_hairpin_tx.actions, error);
5718                 if (ret < 0)
5719                         goto error;
5720         }
5721         /*
5722          * Update the metadata register copy table. If extensive
5723          * metadata feature is enabled and registers are supported
5724          * we might create the extra rte_flow for each unique
5725          * MARK/FLAG action ID.
5726          *
5727          * The table is updated for ingress Flows only, because
5728          * the egress Flows belong to the different device and
5729          * copy table should be updated in peer NIC Rx domain.
5730          */
5731         if (attr_factor.ingress &&
5732             (external || attr_factor.group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5733                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5734                 if (ret)
5735                         goto error;
5736         }
5737         /*
5738          * If the flow is external (from application) OR device is started, then
5739          * the flow will be applied immediately.
5740          */
5741         if (external || dev->data->dev_started) {
5742                 ret = flow_drv_apply(dev, flow, error);
5743                 if (ret < 0)
5744                         goto error;
5745         }
5746         if (list)
5747                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5748                              flow, next);
5749         flow_rxq_flags_set(dev, flow);
5750         rte_free(translated_actions);
5751         /* Nested flow creation index recovery. */
5752         priv->flow_idx = priv->flow_nested_idx;
5753         if (priv->flow_nested_idx)
5754                 priv->flow_nested_idx = 0;
5755         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5756         if (tunnel) {
5757                 flow->tunnel = 1;
5758                 flow->tunnel_id = tunnel->tunnel_id;
5759                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5760                 mlx5_free(default_miss_ctx.queue);
5761         }
5762         return idx;
5763 error:
5764         MLX5_ASSERT(flow);
5765         ret = rte_errno; /* Save rte_errno before cleanup. */
5766         flow_mreg_del_copy_action(dev, flow);
5767         flow_drv_destroy(dev, flow);
5768         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5769         rte_errno = ret; /* Restore rte_errno. */
5770 error_before_flow:
5771         ret = rte_errno;
5772         if (hairpin_id)
5773                 mlx5_flow_id_release(priv->sh->flow_id_pool,
5774                                      hairpin_id);
5775         rte_errno = ret;
5776         priv->flow_idx = priv->flow_nested_idx;
5777         if (priv->flow_nested_idx)
5778                 priv->flow_nested_idx = 0;
5779 error_before_hairpin_split:
5780         rte_free(translated_actions);
5781         return 0;
5782 }
5783
5784 /**
5785  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5786  * incoming packets to table 1.
5787  *
5788  * Other flow rules, requested for group n, will be created in
5789  * e-switch table n+1.
5790  * Jump action to e-switch group n will be created to group n+1.
5791  *
5792  * Used when working in switchdev mode, to utilise advantages of table 1
5793  * and above.
5794  *
5795  * @param dev
5796  *   Pointer to Ethernet device.
5797  *
5798  * @return
5799  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5800  */
5801 struct rte_flow *
5802 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5803 {
5804         const struct rte_flow_attr attr = {
5805                 .group = 0,
5806                 .priority = 0,
5807                 .ingress = 1,
5808                 .egress = 0,
5809                 .transfer = 1,
5810         };
5811         const struct rte_flow_item pattern = {
5812                 .type = RTE_FLOW_ITEM_TYPE_END,
5813         };
5814         struct rte_flow_action_jump jump = {
5815                 .group = 1,
5816         };
5817         const struct rte_flow_action actions[] = {
5818                 {
5819                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5820                         .conf = &jump,
5821                 },
5822                 {
5823                         .type = RTE_FLOW_ACTION_TYPE_END,
5824                 },
5825         };
5826         struct mlx5_priv *priv = dev->data->dev_private;
5827         struct rte_flow_error error;
5828
5829         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5830                                                    &attr, &pattern,
5831                                                    actions, false, &error);
5832 }
5833
5834 /**
5835  * Validate a flow supported by the NIC.
5836  *
5837  * @see rte_flow_validate()
5838  * @see rte_flow_ops
5839  */
5840 int
5841 mlx5_flow_validate(struct rte_eth_dev *dev,
5842                    const struct rte_flow_attr *attr,
5843                    const struct rte_flow_item items[],
5844                    const struct rte_flow_action original_actions[],
5845                    struct rte_flow_error *error)
5846 {
5847         int hairpin_flow;
5848         struct mlx5_translated_shared_action
5849                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5850         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5851         const struct rte_flow_action *actions;
5852         struct rte_flow_action *translated_actions = NULL;
5853         int ret = flow_shared_actions_translate(original_actions,
5854                                                 shared_actions,
5855                                                 &shared_actions_n,
5856                                                 &translated_actions, error);
5857
5858         if (ret)
5859                 return ret;
5860         actions = translated_actions ? translated_actions : original_actions;
5861         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5862         ret = flow_drv_validate(dev, attr, items, actions,
5863                                 true, hairpin_flow, error);
5864         rte_free(translated_actions);
5865         return ret;
5866 }
5867
5868 /**
5869  * Create a flow.
5870  *
5871  * @see rte_flow_create()
5872  * @see rte_flow_ops
5873  */
5874 struct rte_flow *
5875 mlx5_flow_create(struct rte_eth_dev *dev,
5876                  const struct rte_flow_attr *attr,
5877                  const struct rte_flow_item items[],
5878                  const struct rte_flow_action actions[],
5879                  struct rte_flow_error *error)
5880 {
5881         struct mlx5_priv *priv = dev->data->dev_private;
5882
5883         /*
5884          * If the device is not started yet, it is not allowed to created a
5885          * flow from application. PMD default flows and traffic control flows
5886          * are not affected.
5887          */
5888         if (unlikely(!dev->data->dev_started)) {
5889                 DRV_LOG(DEBUG, "port %u is not started when "
5890                         "inserting a flow", dev->data->port_id);
5891                 rte_flow_error_set(error, ENODEV,
5892                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5893                                    NULL,
5894                                    "port not started");
5895                 return NULL;
5896         }
5897
5898         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5899                                   attr, items, actions, true, error);
5900 }
5901
5902 /**
5903  * Destroy a flow in a list.
5904  *
5905  * @param dev
5906  *   Pointer to Ethernet device.
5907  * @param list
5908  *   Pointer to the Indexed flow list. If this parameter NULL,
5909  *   there is no flow removal from the list. Be noted that as
5910  *   flow is add to the indexed list, memory of the indexed
5911  *   list points to maybe changed as flow destroyed.
5912  * @param[in] flow_idx
5913  *   Index of flow to destroy.
5914  */
5915 static void
5916 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5917                   uint32_t flow_idx)
5918 {
5919         struct mlx5_priv *priv = dev->data->dev_private;
5920         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5921         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5922                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5923
5924         if (!flow)
5925                 return;
5926         /*
5927          * Update RX queue flags only if port is started, otherwise it is
5928          * already clean.
5929          */
5930         if (dev->data->dev_started)
5931                 flow_rxq_flags_trim(dev, flow);
5932         if (flow->hairpin_flow_id)
5933                 mlx5_flow_id_release(priv->sh->flow_id_pool,
5934                                      flow->hairpin_flow_id);
5935         flow_drv_destroy(dev, flow);
5936         if (list)
5937                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5938                              flow_idx, flow, next);
5939         flow_mreg_del_copy_action(dev, flow);
5940         if (flow->fdir) {
5941                 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5942                         if (priv_fdir_flow->rix_flow == flow_idx)
5943                                 break;
5944                 }
5945                 if (priv_fdir_flow) {
5946                         LIST_REMOVE(priv_fdir_flow, next);
5947                         mlx5_free(priv_fdir_flow->fdir);
5948                         mlx5_free(priv_fdir_flow);
5949                 }
5950         }
5951         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5952         if (flow->tunnel) {
5953                 struct mlx5_flow_tunnel *tunnel;
5954                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5955                 RTE_VERIFY(tunnel);
5956                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5957                         mlx5_flow_tunnel_free(dev, tunnel);
5958         }
5959 }
5960
5961 /**
5962  * Destroy all flows.
5963  *
5964  * @param dev
5965  *   Pointer to Ethernet device.
5966  * @param list
5967  *   Pointer to the Indexed flow list.
5968  * @param active
5969  *   If flushing is called avtively.
5970  */
5971 void
5972 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5973 {
5974         uint32_t num_flushed = 0;
5975
5976         while (*list) {
5977                 flow_list_destroy(dev, list, *list);
5978                 num_flushed++;
5979         }
5980         if (active) {
5981                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5982                         dev->data->port_id, num_flushed);
5983         }
5984 }
5985
5986 /**
5987  * Remove all flows.
5988  *
5989  * @param dev
5990  *   Pointer to Ethernet device.
5991  * @param list
5992  *   Pointer to the Indexed flow list.
5993  */
5994 void
5995 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
5996 {
5997         struct mlx5_priv *priv = dev->data->dev_private;
5998         struct rte_flow *flow = NULL;
5999         uint32_t idx;
6000
6001         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
6002                       flow, next) {
6003                 flow_drv_remove(dev, flow);
6004                 flow_mreg_stop_copy_action(dev, flow);
6005         }
6006         flow_mreg_del_default_copy_action(dev);
6007         flow_rxq_flags_clear(dev);
6008 }
6009
6010 /**
6011  * Add all flows.
6012  *
6013  * @param dev
6014  *   Pointer to Ethernet device.
6015  * @param list
6016  *   Pointer to the Indexed flow list.
6017  *
6018  * @return
6019  *   0 on success, a negative errno value otherwise and rte_errno is set.
6020  */
6021 int
6022 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
6023 {
6024         struct mlx5_priv *priv = dev->data->dev_private;
6025         struct rte_flow *flow = NULL;
6026         struct rte_flow_error error;
6027         uint32_t idx;
6028         int ret = 0;
6029
6030         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6031         ret = flow_mreg_add_default_copy_action(dev, &error);
6032         if (ret < 0)
6033                 return -rte_errno;
6034         /* Apply Flows created by application. */
6035         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
6036                       flow, next) {
6037                 ret = flow_mreg_start_copy_action(dev, flow);
6038                 if (ret < 0)
6039                         goto error;
6040                 ret = flow_drv_apply(dev, flow, &error);
6041                 if (ret < 0)
6042                         goto error;
6043                 flow_rxq_flags_set(dev, flow);
6044         }
6045         return 0;
6046 error:
6047         ret = rte_errno; /* Save rte_errno before cleanup. */
6048         mlx5_flow_stop(dev, list);
6049         rte_errno = ret; /* Restore rte_errno. */
6050         return -rte_errno;
6051 }
6052
6053 /**
6054  * Stop all default actions for flows.
6055  *
6056  * @param dev
6057  *   Pointer to Ethernet device.
6058  */
6059 void
6060 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6061 {
6062         flow_mreg_del_default_copy_action(dev);
6063         flow_rxq_flags_clear(dev);
6064 }
6065
6066 /**
6067  * Start all default actions for flows.
6068  *
6069  * @param dev
6070  *   Pointer to Ethernet device.
6071  * @return
6072  *   0 on success, a negative errno value otherwise and rte_errno is set.
6073  */
6074 int
6075 mlx5_flow_start_default(struct rte_eth_dev *dev)
6076 {
6077         struct rte_flow_error error;
6078
6079         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6080         return flow_mreg_add_default_copy_action(dev, &error);
6081 }
6082
6083 /**
6084  * Allocate intermediate resources for flow creation.
6085  *
6086  * @param dev
6087  *   Pointer to Ethernet device.
6088  */
6089 void
6090 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
6091 {
6092         struct mlx5_priv *priv = dev->data->dev_private;
6093
6094         if (!priv->inter_flows) {
6095                 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO,
6096                                     MLX5_NUM_MAX_DEV_FLOWS *
6097                                     sizeof(struct mlx5_flow) +
6098                                     (sizeof(struct mlx5_flow_rss_desc) +
6099                                     sizeof(uint16_t) * UINT16_MAX) * 2, 0,
6100                                     SOCKET_ID_ANY);
6101                 if (!priv->inter_flows) {
6102                         DRV_LOG(ERR, "can't allocate intermediate memory.");
6103                         return;
6104                 }
6105         }
6106         priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
6107                          [MLX5_NUM_MAX_DEV_FLOWS];
6108         /* Reset the index. */
6109         priv->flow_idx = 0;
6110         priv->flow_nested_idx = 0;
6111 }
6112
6113 /**
6114  * Free intermediate resources for flows.
6115  *
6116  * @param dev
6117  *   Pointer to Ethernet device.
6118  */
6119 void
6120 mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
6121 {
6122         struct mlx5_priv *priv = dev->data->dev_private;
6123
6124         mlx5_free(priv->inter_flows);
6125         priv->inter_flows = NULL;
6126 }
6127
6128 /**
6129  * Verify the flow list is empty
6130  *
6131  * @param dev
6132  *  Pointer to Ethernet device.
6133  *
6134  * @return the number of flows not released.
6135  */
6136 int
6137 mlx5_flow_verify(struct rte_eth_dev *dev)
6138 {
6139         struct mlx5_priv *priv = dev->data->dev_private;
6140         struct rte_flow *flow;
6141         uint32_t idx;
6142         int ret = 0;
6143
6144         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6145                       flow, next) {
6146                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6147                         dev->data->port_id, (void *)flow);
6148                 ++ret;
6149         }
6150         return ret;
6151 }
6152
6153 /**
6154  * Enable default hairpin egress flow.
6155  *
6156  * @param dev
6157  *   Pointer to Ethernet device.
6158  * @param queue
6159  *   The queue index.
6160  *
6161  * @return
6162  *   0 on success, a negative errno value otherwise and rte_errno is set.
6163  */
6164 int
6165 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6166                             uint32_t queue)
6167 {
6168         struct mlx5_priv *priv = dev->data->dev_private;
6169         const struct rte_flow_attr attr = {
6170                 .egress = 1,
6171                 .priority = 0,
6172         };
6173         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6174                 .queue = queue,
6175         };
6176         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6177                 .queue = UINT32_MAX,
6178         };
6179         struct rte_flow_item items[] = {
6180                 {
6181                         .type = (enum rte_flow_item_type)
6182                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6183                         .spec = &queue_spec,
6184                         .last = NULL,
6185                         .mask = &queue_mask,
6186                 },
6187                 {
6188                         .type = RTE_FLOW_ITEM_TYPE_END,
6189                 },
6190         };
6191         struct rte_flow_action_jump jump = {
6192                 .group = MLX5_HAIRPIN_TX_TABLE,
6193         };
6194         struct rte_flow_action actions[2];
6195         uint32_t flow_idx;
6196         struct rte_flow_error error;
6197
6198         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6199         actions[0].conf = &jump;
6200         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6201         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6202                                 &attr, items, actions, false, &error);
6203         if (!flow_idx) {
6204                 DRV_LOG(DEBUG,
6205                         "Failed to create ctrl flow: rte_errno(%d),"
6206                         " type(%d), message(%s)",
6207                         rte_errno, error.type,
6208                         error.message ? error.message : " (no stated reason)");
6209                 return -rte_errno;
6210         }
6211         return 0;
6212 }
6213
6214 /**
6215  * Enable a control flow configured from the control plane.
6216  *
6217  * @param dev
6218  *   Pointer to Ethernet device.
6219  * @param eth_spec
6220  *   An Ethernet flow spec to apply.
6221  * @param eth_mask
6222  *   An Ethernet flow mask to apply.
6223  * @param vlan_spec
6224  *   A VLAN flow spec to apply.
6225  * @param vlan_mask
6226  *   A VLAN flow mask to apply.
6227  *
6228  * @return
6229  *   0 on success, a negative errno value otherwise and rte_errno is set.
6230  */
6231 int
6232 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6233                     struct rte_flow_item_eth *eth_spec,
6234                     struct rte_flow_item_eth *eth_mask,
6235                     struct rte_flow_item_vlan *vlan_spec,
6236                     struct rte_flow_item_vlan *vlan_mask)
6237 {
6238         struct mlx5_priv *priv = dev->data->dev_private;
6239         const struct rte_flow_attr attr = {
6240                 .ingress = 1,
6241                 .priority = MLX5_FLOW_PRIO_RSVD,
6242         };
6243         struct rte_flow_item items[] = {
6244                 {
6245                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6246                         .spec = eth_spec,
6247                         .last = NULL,
6248                         .mask = eth_mask,
6249                 },
6250                 {
6251                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6252                                               RTE_FLOW_ITEM_TYPE_END,
6253                         .spec = vlan_spec,
6254                         .last = NULL,
6255                         .mask = vlan_mask,
6256                 },
6257                 {
6258                         .type = RTE_FLOW_ITEM_TYPE_END,
6259                 },
6260         };
6261         uint16_t queue[priv->reta_idx_n];
6262         struct rte_flow_action_rss action_rss = {
6263                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6264                 .level = 0,
6265                 .types = priv->rss_conf.rss_hf,
6266                 .key_len = priv->rss_conf.rss_key_len,
6267                 .queue_num = priv->reta_idx_n,
6268                 .key = priv->rss_conf.rss_key,
6269                 .queue = queue,
6270         };
6271         struct rte_flow_action actions[] = {
6272                 {
6273                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6274                         .conf = &action_rss,
6275                 },
6276                 {
6277                         .type = RTE_FLOW_ACTION_TYPE_END,
6278                 },
6279         };
6280         uint32_t flow_idx;
6281         struct rte_flow_error error;
6282         unsigned int i;
6283
6284         if (!priv->reta_idx_n || !priv->rxqs_n) {
6285                 return 0;
6286         }
6287         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6288                 action_rss.types = 0;
6289         for (i = 0; i != priv->reta_idx_n; ++i)
6290                 queue[i] = (*priv->reta_idx)[i];
6291         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6292                                 &attr, items, actions, false, &error);
6293         if (!flow_idx)
6294                 return -rte_errno;
6295         return 0;
6296 }
6297
6298 /**
6299  * Enable a flow control configured from the control plane.
6300  *
6301  * @param dev
6302  *   Pointer to Ethernet device.
6303  * @param eth_spec
6304  *   An Ethernet flow spec to apply.
6305  * @param eth_mask
6306  *   An Ethernet flow mask to apply.
6307  *
6308  * @return
6309  *   0 on success, a negative errno value otherwise and rte_errno is set.
6310  */
6311 int
6312 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6313                struct rte_flow_item_eth *eth_spec,
6314                struct rte_flow_item_eth *eth_mask)
6315 {
6316         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6317 }
6318
6319 /**
6320  * Create default miss flow rule matching lacp traffic
6321  *
6322  * @param dev
6323  *   Pointer to Ethernet device.
6324  * @param eth_spec
6325  *   An Ethernet flow spec to apply.
6326  *
6327  * @return
6328  *   0 on success, a negative errno value otherwise and rte_errno is set.
6329  */
6330 int
6331 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6332 {
6333         struct mlx5_priv *priv = dev->data->dev_private;
6334         /*
6335          * The LACP matching is done by only using ether type since using
6336          * a multicast dst mac causes kernel to give low priority to this flow.
6337          */
6338         static const struct rte_flow_item_eth lacp_spec = {
6339                 .type = RTE_BE16(0x8809),
6340         };
6341         static const struct rte_flow_item_eth lacp_mask = {
6342                 .type = 0xffff,
6343         };
6344         const struct rte_flow_attr attr = {
6345                 .ingress = 1,
6346         };
6347         struct rte_flow_item items[] = {
6348                 {
6349                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6350                         .spec = &lacp_spec,
6351                         .mask = &lacp_mask,
6352                 },
6353                 {
6354                         .type = RTE_FLOW_ITEM_TYPE_END,
6355                 },
6356         };
6357         struct rte_flow_action actions[] = {
6358                 {
6359                         .type = (enum rte_flow_action_type)
6360                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6361                 },
6362                 {
6363                         .type = RTE_FLOW_ACTION_TYPE_END,
6364                 },
6365         };
6366         struct rte_flow_error error;
6367         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6368                                 &attr, items, actions, false, &error);
6369
6370         if (!flow_idx)
6371                 return -rte_errno;
6372         return 0;
6373 }
6374
6375 /**
6376  * Destroy a flow.
6377  *
6378  * @see rte_flow_destroy()
6379  * @see rte_flow_ops
6380  */
6381 int
6382 mlx5_flow_destroy(struct rte_eth_dev *dev,
6383                   struct rte_flow *flow,
6384                   struct rte_flow_error *error __rte_unused)
6385 {
6386         struct mlx5_priv *priv = dev->data->dev_private;
6387
6388         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6389         return 0;
6390 }
6391
6392 /**
6393  * Destroy all flows.
6394  *
6395  * @see rte_flow_flush()
6396  * @see rte_flow_ops
6397  */
6398 int
6399 mlx5_flow_flush(struct rte_eth_dev *dev,
6400                 struct rte_flow_error *error __rte_unused)
6401 {
6402         struct mlx5_priv *priv = dev->data->dev_private;
6403
6404         mlx5_flow_list_flush(dev, &priv->flows, false);
6405         return 0;
6406 }
6407
6408 /**
6409  * Isolated mode.
6410  *
6411  * @see rte_flow_isolate()
6412  * @see rte_flow_ops
6413  */
6414 int
6415 mlx5_flow_isolate(struct rte_eth_dev *dev,
6416                   int enable,
6417                   struct rte_flow_error *error)
6418 {
6419         struct mlx5_priv *priv = dev->data->dev_private;
6420
6421         if (dev->data->dev_started) {
6422                 rte_flow_error_set(error, EBUSY,
6423                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6424                                    NULL,
6425                                    "port must be stopped first");
6426                 return -rte_errno;
6427         }
6428         priv->isolated = !!enable;
6429         if (enable)
6430                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6431         else
6432                 dev->dev_ops = &mlx5_os_dev_ops;
6433
6434         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6435         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6436
6437         return 0;
6438 }
6439
6440 /**
6441  * Query a flow.
6442  *
6443  * @see rte_flow_query()
6444  * @see rte_flow_ops
6445  */
6446 static int
6447 flow_drv_query(struct rte_eth_dev *dev,
6448                uint32_t flow_idx,
6449                const struct rte_flow_action *actions,
6450                void *data,
6451                struct rte_flow_error *error)
6452 {
6453         struct mlx5_priv *priv = dev->data->dev_private;
6454         const struct mlx5_flow_driver_ops *fops;
6455         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6456                                                [MLX5_IPOOL_RTE_FLOW],
6457                                                flow_idx);
6458         enum mlx5_flow_drv_type ftype;
6459
6460         if (!flow) {
6461                 return rte_flow_error_set(error, ENOENT,
6462                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6463                           NULL,
6464                           "invalid flow handle");
6465         }
6466         ftype = flow->drv_type;
6467         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6468         fops = flow_get_drv_ops(ftype);
6469
6470         return fops->query(dev, flow, actions, data, error);
6471 }
6472
6473 /**
6474  * Query a flow.
6475  *
6476  * @see rte_flow_query()
6477  * @see rte_flow_ops
6478  */
6479 int
6480 mlx5_flow_query(struct rte_eth_dev *dev,
6481                 struct rte_flow *flow,
6482                 const struct rte_flow_action *actions,
6483                 void *data,
6484                 struct rte_flow_error *error)
6485 {
6486         int ret;
6487
6488         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6489                              error);
6490         if (ret < 0)
6491                 return ret;
6492         return 0;
6493 }
6494
6495 /**
6496  * Convert a flow director filter to a generic flow.
6497  *
6498  * @param dev
6499  *   Pointer to Ethernet device.
6500  * @param fdir_filter
6501  *   Flow director filter to add.
6502  * @param attributes
6503  *   Generic flow parameters structure.
6504  *
6505  * @return
6506  *   0 on success, a negative errno value otherwise and rte_errno is set.
6507  */
6508 static int
6509 flow_fdir_filter_convert(struct rte_eth_dev *dev,
6510                          const struct rte_eth_fdir_filter *fdir_filter,
6511                          struct mlx5_fdir *attributes)
6512 {
6513         struct mlx5_priv *priv = dev->data->dev_private;
6514         const struct rte_eth_fdir_input *input = &fdir_filter->input;
6515         const struct rte_eth_fdir_masks *mask =
6516                 &dev->data->dev_conf.fdir_conf.mask;
6517
6518         /* Validate queue number. */
6519         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
6520                 DRV_LOG(ERR, "port %u invalid queue number %d",
6521                         dev->data->port_id, fdir_filter->action.rx_queue);
6522                 rte_errno = EINVAL;
6523                 return -rte_errno;
6524         }
6525         attributes->attr.ingress = 1;
6526         attributes->items[0] = (struct rte_flow_item) {
6527                 .type = RTE_FLOW_ITEM_TYPE_ETH,
6528                 .spec = &attributes->l2,
6529                 .mask = &attributes->l2_mask,
6530         };
6531         switch (fdir_filter->action.behavior) {
6532         case RTE_ETH_FDIR_ACCEPT:
6533                 attributes->actions[0] = (struct rte_flow_action){
6534                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
6535                         .conf = &attributes->queue,
6536                 };
6537                 break;
6538         case RTE_ETH_FDIR_REJECT:
6539                 attributes->actions[0] = (struct rte_flow_action){
6540                         .type = RTE_FLOW_ACTION_TYPE_DROP,
6541                 };
6542                 break;
6543         default:
6544                 DRV_LOG(ERR, "port %u invalid behavior %d",
6545                         dev->data->port_id,
6546                         fdir_filter->action.behavior);
6547                 rte_errno = ENOTSUP;
6548                 return -rte_errno;
6549         }
6550         attributes->queue.index = fdir_filter->action.rx_queue;
6551         /* Handle L3. */
6552         switch (fdir_filter->input.flow_type) {
6553         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
6554         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
6555         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
6556                 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
6557                         .src_addr = input->flow.ip4_flow.src_ip,
6558                         .dst_addr = input->flow.ip4_flow.dst_ip,
6559                         .time_to_live = input->flow.ip4_flow.ttl,
6560                         .type_of_service = input->flow.ip4_flow.tos,
6561                 };
6562                 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
6563                         .src_addr = mask->ipv4_mask.src_ip,
6564                         .dst_addr = mask->ipv4_mask.dst_ip,
6565                         .time_to_live = mask->ipv4_mask.ttl,
6566                         .type_of_service = mask->ipv4_mask.tos,
6567                         .next_proto_id = mask->ipv4_mask.proto,
6568                 };
6569                 attributes->items[1] = (struct rte_flow_item){
6570                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
6571                         .spec = &attributes->l3,
6572                         .mask = &attributes->l3_mask,
6573                 };
6574                 break;
6575         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
6576         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
6577         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
6578                 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
6579                         .hop_limits = input->flow.ipv6_flow.hop_limits,
6580                         .proto = input->flow.ipv6_flow.proto,
6581                 };
6582
6583                 memcpy(attributes->l3.ipv6.hdr.src_addr,
6584                        input->flow.ipv6_flow.src_ip,
6585                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
6586                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
6587                        input->flow.ipv6_flow.dst_ip,
6588                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
6589                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
6590                        mask->ipv6_mask.src_ip,
6591                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
6592                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
6593                        mask->ipv6_mask.dst_ip,
6594                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
6595                 attributes->items[1] = (struct rte_flow_item){
6596                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
6597                         .spec = &attributes->l3,
6598                         .mask = &attributes->l3_mask,
6599                 };
6600                 break;
6601         default:
6602                 DRV_LOG(ERR, "port %u invalid flow type%d",
6603                         dev->data->port_id, fdir_filter->input.flow_type);
6604                 rte_errno = ENOTSUP;
6605                 return -rte_errno;
6606         }
6607         /* Handle L4. */
6608         switch (fdir_filter->input.flow_type) {
6609         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
6610                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
6611                         .src_port = input->flow.udp4_flow.src_port,
6612                         .dst_port = input->flow.udp4_flow.dst_port,
6613                 };
6614                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
6615                         .src_port = mask->src_port_mask,
6616                         .dst_port = mask->dst_port_mask,
6617                 };
6618                 attributes->items[2] = (struct rte_flow_item){
6619                         .type = RTE_FLOW_ITEM_TYPE_UDP,
6620                         .spec = &attributes->l4,
6621                         .mask = &attributes->l4_mask,
6622                 };
6623                 break;
6624         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
6625                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
6626                         .src_port = input->flow.tcp4_flow.src_port,
6627                         .dst_port = input->flow.tcp4_flow.dst_port,
6628                 };
6629                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
6630                         .src_port = mask->src_port_mask,
6631                         .dst_port = mask->dst_port_mask,
6632                 };
6633                 attributes->items[2] = (struct rte_flow_item){
6634                         .type = RTE_FLOW_ITEM_TYPE_TCP,
6635                         .spec = &attributes->l4,
6636                         .mask = &attributes->l4_mask,
6637                 };
6638                 break;
6639         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
6640                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
6641                         .src_port = input->flow.udp6_flow.src_port,
6642                         .dst_port = input->flow.udp6_flow.dst_port,
6643                 };
6644                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
6645                         .src_port = mask->src_port_mask,
6646                         .dst_port = mask->dst_port_mask,
6647                 };
6648                 attributes->items[2] = (struct rte_flow_item){
6649                         .type = RTE_FLOW_ITEM_TYPE_UDP,
6650                         .spec = &attributes->l4,
6651                         .mask = &attributes->l4_mask,
6652                 };
6653                 break;
6654         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
6655                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
6656                         .src_port = input->flow.tcp6_flow.src_port,
6657                         .dst_port = input->flow.tcp6_flow.dst_port,
6658                 };
6659                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
6660                         .src_port = mask->src_port_mask,
6661                         .dst_port = mask->dst_port_mask,
6662                 };
6663                 attributes->items[2] = (struct rte_flow_item){
6664                         .type = RTE_FLOW_ITEM_TYPE_TCP,
6665                         .spec = &attributes->l4,
6666                         .mask = &attributes->l4_mask,
6667                 };
6668                 break;
6669         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
6670         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
6671                 break;
6672         default:
6673                 DRV_LOG(ERR, "port %u invalid flow type%d",
6674                         dev->data->port_id, fdir_filter->input.flow_type);
6675                 rte_errno = ENOTSUP;
6676                 return -rte_errno;
6677         }
6678         return 0;
6679 }
6680
6681 #define FLOW_FDIR_CMP(f1, f2, fld) \
6682         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
6683
6684 /**
6685  * Compare two FDIR flows. If items and actions are identical, the two flows are
6686  * regarded as same.
6687  *
6688  * @param dev
6689  *   Pointer to Ethernet device.
6690  * @param f1
6691  *   FDIR flow to compare.
6692  * @param f2
6693  *   FDIR flow to compare.
6694  *
6695  * @return
6696  *   Zero on match, 1 otherwise.
6697  */
6698 static int
6699 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
6700 {
6701         if (FLOW_FDIR_CMP(f1, f2, attr) ||
6702             FLOW_FDIR_CMP(f1, f2, l2) ||
6703             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
6704             FLOW_FDIR_CMP(f1, f2, l3) ||
6705             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
6706             FLOW_FDIR_CMP(f1, f2, l4) ||
6707             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
6708             FLOW_FDIR_CMP(f1, f2, actions[0].type))
6709                 return 1;
6710         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
6711             FLOW_FDIR_CMP(f1, f2, queue))
6712                 return 1;
6713         return 0;
6714 }
6715
6716 /**
6717  * Search device flow list to find out a matched FDIR flow.
6718  *
6719  * @param dev
6720  *   Pointer to Ethernet device.
6721  * @param fdir_flow
6722  *   FDIR flow to lookup.
6723  *
6724  * @return
6725  *   Index of flow if found, 0 otherwise.
6726  */
6727 static uint32_t
6728 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
6729 {
6730         struct mlx5_priv *priv = dev->data->dev_private;
6731         uint32_t flow_idx = 0;
6732         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6733
6734         MLX5_ASSERT(fdir_flow);
6735         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
6736                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
6737                         DRV_LOG(DEBUG, "port %u found FDIR flow %u",
6738                                 dev->data->port_id, flow_idx);
6739                         flow_idx = priv_fdir_flow->rix_flow;
6740                         break;
6741                 }
6742         }
6743         return flow_idx;
6744 }
6745
6746 /**
6747  * Add new flow director filter and store it in list.
6748  *
6749  * @param dev
6750  *   Pointer to Ethernet device.
6751  * @param fdir_filter
6752  *   Flow director filter to add.
6753  *
6754  * @return
6755  *   0 on success, a negative errno value otherwise and rte_errno is set.
6756  */
6757 static int
6758 flow_fdir_filter_add(struct rte_eth_dev *dev,
6759                      const struct rte_eth_fdir_filter *fdir_filter)
6760 {
6761         struct mlx5_priv *priv = dev->data->dev_private;
6762         struct mlx5_fdir *fdir_flow;
6763         struct rte_flow *flow;
6764         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6765         uint32_t flow_idx;
6766         int ret;
6767
6768         fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0,
6769                                 SOCKET_ID_ANY);
6770         if (!fdir_flow) {
6771                 rte_errno = ENOMEM;
6772                 return -rte_errno;
6773         }
6774         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
6775         if (ret)
6776                 goto error;
6777         flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
6778         if (flow_idx) {
6779                 rte_errno = EEXIST;
6780                 goto error;
6781         }
6782         priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO,
6783                                      sizeof(struct mlx5_fdir_flow),
6784                                      0, SOCKET_ID_ANY);
6785         if (!priv_fdir_flow) {
6786                 rte_errno = ENOMEM;
6787                 goto error;
6788         }
6789         flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
6790                                     fdir_flow->items, fdir_flow->actions, true,
6791                                     NULL);
6792         flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6793         if (!flow)
6794                 goto error;
6795         flow->fdir = 1;
6796         priv_fdir_flow->fdir = fdir_flow;
6797         priv_fdir_flow->rix_flow = flow_idx;
6798         LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
6799         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
6800                 dev->data->port_id, (void *)flow);
6801         return 0;
6802 error:
6803         mlx5_free(priv_fdir_flow);
6804         mlx5_free(fdir_flow);
6805         return -rte_errno;
6806 }
6807
6808 /**
6809  * Delete specific filter.
6810  *
6811  * @param dev
6812  *   Pointer to Ethernet device.
6813  * @param fdir_filter
6814  *   Filter to be deleted.
6815  *
6816  * @return
6817  *   0 on success, a negative errno value otherwise and rte_errno is set.
6818  */
6819 static int
6820 flow_fdir_filter_delete(struct rte_eth_dev *dev,
6821                         const struct rte_eth_fdir_filter *fdir_filter)
6822 {
6823         struct mlx5_priv *priv = dev->data->dev_private;
6824         uint32_t flow_idx;
6825         struct mlx5_fdir fdir_flow = {
6826                 .attr.group = 0,
6827         };
6828         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6829         int ret;
6830
6831         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
6832         if (ret)
6833                 return -rte_errno;
6834         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
6835                 /* Find the fdir in priv list */
6836                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
6837                         break;
6838         }
6839         if (!priv_fdir_flow)
6840                 return 0;
6841         LIST_REMOVE(priv_fdir_flow, next);
6842         flow_idx = priv_fdir_flow->rix_flow;
6843         flow_list_destroy(dev, &priv->flows, flow_idx);
6844         mlx5_free(priv_fdir_flow->fdir);
6845         mlx5_free(priv_fdir_flow);
6846         DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
6847                 dev->data->port_id, flow_idx);
6848         return 0;
6849 }
6850
6851 /**
6852  * Update queue for specific filter.
6853  *
6854  * @param dev
6855  *   Pointer to Ethernet device.
6856  * @param fdir_filter
6857  *   Filter to be updated.
6858  *
6859  * @return
6860  *   0 on success, a negative errno value otherwise and rte_errno is set.
6861  */
6862 static int
6863 flow_fdir_filter_update(struct rte_eth_dev *dev,
6864                         const struct rte_eth_fdir_filter *fdir_filter)
6865 {
6866         int ret;
6867
6868         ret = flow_fdir_filter_delete(dev, fdir_filter);
6869         if (ret)
6870                 return ret;
6871         return flow_fdir_filter_add(dev, fdir_filter);
6872 }
6873
6874 /**
6875  * Flush all filters.
6876  *
6877  * @param dev
6878  *   Pointer to Ethernet device.
6879  */
6880 static void
6881 flow_fdir_filter_flush(struct rte_eth_dev *dev)
6882 {
6883         struct mlx5_priv *priv = dev->data->dev_private;
6884         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6885
6886         while (!LIST_EMPTY(&priv->fdir_flows)) {
6887                 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
6888                 LIST_REMOVE(priv_fdir_flow, next);
6889                 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
6890                 mlx5_free(priv_fdir_flow->fdir);
6891                 mlx5_free(priv_fdir_flow);
6892         }
6893 }
6894
6895 /**
6896  * Get flow director information.
6897  *
6898  * @param dev
6899  *   Pointer to Ethernet device.
6900  * @param[out] fdir_info
6901  *   Resulting flow director information.
6902  */
6903 static void
6904 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
6905 {
6906         struct rte_eth_fdir_masks *mask =
6907                 &dev->data->dev_conf.fdir_conf.mask;
6908
6909         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
6910         fdir_info->guarant_spc = 0;
6911         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
6912         fdir_info->max_flexpayload = 0;
6913         fdir_info->flow_types_mask[0] = 0;
6914         fdir_info->flex_payload_unit = 0;
6915         fdir_info->max_flex_payload_segment_num = 0;
6916         fdir_info->flex_payload_limit = 0;
6917         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
6918 }
6919
6920 /**
6921  * Deal with flow director operations.
6922  *
6923  * @param dev
6924  *   Pointer to Ethernet device.
6925  * @param filter_op
6926  *   Operation to perform.
6927  * @param arg
6928  *   Pointer to operation-specific structure.
6929  *
6930  * @return
6931  *   0 on success, a negative errno value otherwise and rte_errno is set.
6932  */
6933 static int
6934 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
6935                     void *arg)
6936 {
6937         enum rte_fdir_mode fdir_mode =
6938                 dev->data->dev_conf.fdir_conf.mode;
6939
6940         if (filter_op == RTE_ETH_FILTER_NOP)
6941                 return 0;
6942         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
6943             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
6944                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
6945                         dev->data->port_id, fdir_mode);
6946                 rte_errno = EINVAL;
6947                 return -rte_errno;
6948         }
6949         switch (filter_op) {
6950         case RTE_ETH_FILTER_ADD:
6951                 return flow_fdir_filter_add(dev, arg);
6952         case RTE_ETH_FILTER_UPDATE:
6953                 return flow_fdir_filter_update(dev, arg);
6954         case RTE_ETH_FILTER_DELETE:
6955                 return flow_fdir_filter_delete(dev, arg);
6956         case RTE_ETH_FILTER_FLUSH:
6957                 flow_fdir_filter_flush(dev);
6958                 break;
6959         case RTE_ETH_FILTER_INFO:
6960                 flow_fdir_info_get(dev, arg);
6961                 break;
6962         default:
6963                 DRV_LOG(DEBUG, "port %u unknown operation %u",
6964                         dev->data->port_id, filter_op);
6965                 rte_errno = EINVAL;
6966                 return -rte_errno;
6967         }
6968         return 0;
6969 }
6970
6971 /**
6972  * Manage filter operations.
6973  *
6974  * @param dev
6975  *   Pointer to Ethernet device structure.
6976  * @param filter_type
6977  *   Filter type.
6978  * @param filter_op
6979  *   Operation to perform.
6980  * @param arg
6981  *   Pointer to operation-specific structure.
6982  *
6983  * @return
6984  *   0 on success, a negative errno value otherwise and rte_errno is set.
6985  */
6986 int
6987 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6988                      enum rte_filter_type filter_type,
6989                      enum rte_filter_op filter_op,
6990                      void *arg)
6991 {
6992         switch (filter_type) {
6993         case RTE_ETH_FILTER_GENERIC:
6994                 if (filter_op != RTE_ETH_FILTER_GET) {
6995                         rte_errno = EINVAL;
6996                         return -rte_errno;
6997                 }
6998                 *(const void **)arg = &mlx5_flow_ops;
6999                 return 0;
7000         case RTE_ETH_FILTER_FDIR:
7001                 return flow_fdir_ctrl_func(dev, filter_op, arg);
7002         default:
7003                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
7004                         dev->data->port_id, filter_type);
7005                 rte_errno = ENOTSUP;
7006                 return -rte_errno;
7007         }
7008         return 0;
7009 }
7010
7011 /**
7012  * Create the needed meter and suffix tables.
7013  *
7014  * @param[in] dev
7015  *   Pointer to Ethernet device.
7016  * @param[in] fm
7017  *   Pointer to the flow meter.
7018  *
7019  * @return
7020  *   Pointer to table set on success, NULL otherwise.
7021  */
7022 struct mlx5_meter_domains_infos *
7023 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7024                           const struct mlx5_flow_meter *fm)
7025 {
7026         const struct mlx5_flow_driver_ops *fops;
7027
7028         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7029         return fops->create_mtr_tbls(dev, fm);
7030 }
7031
7032 /**
7033  * Destroy the meter table set.
7034  *
7035  * @param[in] dev
7036  *   Pointer to Ethernet device.
7037  * @param[in] tbl
7038  *   Pointer to the meter table set.
7039  *
7040  * @return
7041  *   0 on success.
7042  */
7043 int
7044 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7045                            struct mlx5_meter_domains_infos *tbls)
7046 {
7047         const struct mlx5_flow_driver_ops *fops;
7048
7049         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7050         return fops->destroy_mtr_tbls(dev, tbls);
7051 }
7052
7053 /**
7054  * Create policer rules.
7055  *
7056  * @param[in] dev
7057  *   Pointer to Ethernet device.
7058  * @param[in] fm
7059  *   Pointer to flow meter structure.
7060  * @param[in] attr
7061  *   Pointer to flow attributes.
7062  *
7063  * @return
7064  *   0 on success, -1 otherwise.
7065  */
7066 int
7067 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
7068                                struct mlx5_flow_meter *fm,
7069                                const struct rte_flow_attr *attr)
7070 {
7071         const struct mlx5_flow_driver_ops *fops;
7072
7073         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7074         return fops->create_policer_rules(dev, fm, attr);
7075 }
7076
7077 /**
7078  * Destroy policer rules.
7079  *
7080  * @param[in] fm
7081  *   Pointer to flow meter structure.
7082  * @param[in] attr
7083  *   Pointer to flow attributes.
7084  *
7085  * @return
7086  *   0 on success, -1 otherwise.
7087  */
7088 int
7089 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
7090                                 struct mlx5_flow_meter *fm,
7091                                 const struct rte_flow_attr *attr)
7092 {
7093         const struct mlx5_flow_driver_ops *fops;
7094
7095         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7096         return fops->destroy_policer_rules(dev, fm, attr);
7097 }
7098
7099 /**
7100  * Allocate a counter.
7101  *
7102  * @param[in] dev
7103  *   Pointer to Ethernet device structure.
7104  *
7105  * @return
7106  *   Index to allocated counter  on success, 0 otherwise.
7107  */
7108 uint32_t
7109 mlx5_counter_alloc(struct rte_eth_dev *dev)
7110 {
7111         const struct mlx5_flow_driver_ops *fops;
7112         struct rte_flow_attr attr = { .transfer = 0 };
7113
7114         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7115                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7116                 return fops->counter_alloc(dev);
7117         }
7118         DRV_LOG(ERR,
7119                 "port %u counter allocate is not supported.",
7120                  dev->data->port_id);
7121         return 0;
7122 }
7123
7124 /**
7125  * Free a counter.
7126  *
7127  * @param[in] dev
7128  *   Pointer to Ethernet device structure.
7129  * @param[in] cnt
7130  *   Index to counter to be free.
7131  */
7132 void
7133 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7134 {
7135         const struct mlx5_flow_driver_ops *fops;
7136         struct rte_flow_attr attr = { .transfer = 0 };
7137
7138         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7139                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7140                 fops->counter_free(dev, cnt);
7141                 return;
7142         }
7143         DRV_LOG(ERR,
7144                 "port %u counter free is not supported.",
7145                  dev->data->port_id);
7146 }
7147
7148 /**
7149  * Query counter statistics.
7150  *
7151  * @param[in] dev
7152  *   Pointer to Ethernet device structure.
7153  * @param[in] cnt
7154  *   Index to counter to query.
7155  * @param[in] clear
7156  *   Set to clear counter statistics.
7157  * @param[out] pkts
7158  *   The counter hits packets number to save.
7159  * @param[out] bytes
7160  *   The counter hits bytes number to save.
7161  *
7162  * @return
7163  *   0 on success, a negative errno value otherwise.
7164  */
7165 int
7166 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7167                    bool clear, uint64_t *pkts, uint64_t *bytes)
7168 {
7169         const struct mlx5_flow_driver_ops *fops;
7170         struct rte_flow_attr attr = { .transfer = 0 };
7171
7172         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7173                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7174                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7175         }
7176         DRV_LOG(ERR,
7177                 "port %u counter query is not supported.",
7178                  dev->data->port_id);
7179         return -ENOTSUP;
7180 }
7181
7182 /**
7183  * Allocate a new memory for the counter values wrapped by all the needed
7184  * management.
7185  *
7186  * @param[in] sh
7187  *   Pointer to mlx5_dev_ctx_shared object.
7188  *
7189  * @return
7190  *   0 on success, a negative errno value otherwise.
7191  */
7192 static int
7193 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7194 {
7195         struct mlx5_devx_mkey_attr mkey_attr;
7196         struct mlx5_counter_stats_mem_mng *mem_mng;
7197         volatile struct flow_counter_stats *raw_data;
7198         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7199         int size = (sizeof(struct flow_counter_stats) *
7200                         MLX5_COUNTERS_PER_POOL +
7201                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7202                         sizeof(struct mlx5_counter_stats_mem_mng);
7203         size_t pgsize = rte_mem_page_size();
7204         uint8_t *mem;
7205         int i;
7206
7207         if (pgsize == (size_t)-1) {
7208                 DRV_LOG(ERR, "Failed to get mem page size");
7209                 rte_errno = ENOMEM;
7210                 return -ENOMEM;
7211         }
7212         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7213         if (!mem) {
7214                 rte_errno = ENOMEM;
7215                 return -ENOMEM;
7216         }
7217         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7218         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7219         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
7220                                                  IBV_ACCESS_LOCAL_WRITE);
7221         if (!mem_mng->umem) {
7222                 rte_errno = errno;
7223                 mlx5_free(mem);
7224                 return -rte_errno;
7225         }
7226         mkey_attr.addr = (uintptr_t)mem;
7227         mkey_attr.size = size;
7228         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7229         mkey_attr.pd = sh->pdn;
7230         mkey_attr.log_entity_size = 0;
7231         mkey_attr.pg_access = 0;
7232         mkey_attr.klm_array = NULL;
7233         mkey_attr.klm_num = 0;
7234         mkey_attr.relaxed_ordering = sh->cmng.relaxed_ordering;
7235         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7236         if (!mem_mng->dm) {
7237                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
7238                 rte_errno = errno;
7239                 mlx5_free(mem);
7240                 return -rte_errno;
7241         }
7242         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7243         raw_data = (volatile struct flow_counter_stats *)mem;
7244         for (i = 0; i < raws_n; ++i) {
7245                 mem_mng->raws[i].mem_mng = mem_mng;
7246                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7247         }
7248         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7249                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7250                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7251                                  next);
7252         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7253         sh->cmng.mem_mng = mem_mng;
7254         return 0;
7255 }
7256
7257 /**
7258  * Set the statistic memory to the new counter pool.
7259  *
7260  * @param[in] sh
7261  *   Pointer to mlx5_dev_ctx_shared object.
7262  * @param[in] pool
7263  *   Pointer to the pool to set the statistic memory.
7264  *
7265  * @return
7266  *   0 on success, a negative errno value otherwise.
7267  */
7268 static int
7269 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7270                                struct mlx5_flow_counter_pool *pool)
7271 {
7272         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7273         /* Resize statistic memory once used out. */
7274         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7275             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7276                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7277                 return -1;
7278         }
7279         rte_spinlock_lock(&pool->sl);
7280         pool->raw = cmng->mem_mng->raws + pool->index %
7281                     MLX5_CNT_CONTAINER_RESIZE;
7282         rte_spinlock_unlock(&pool->sl);
7283         pool->raw_hw = NULL;
7284         return 0;
7285 }
7286
7287 #define MLX5_POOL_QUERY_FREQ_US 1000000
7288
7289 /**
7290  * Set the periodic procedure for triggering asynchronous batch queries for all
7291  * the counter pools.
7292  *
7293  * @param[in] sh
7294  *   Pointer to mlx5_dev_ctx_shared object.
7295  */
7296 void
7297 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7298 {
7299         uint32_t pools_n, us;
7300
7301         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7302         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7303         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7304         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7305                 sh->cmng.query_thread_on = 0;
7306                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7307         } else {
7308                 sh->cmng.query_thread_on = 1;
7309         }
7310 }
7311
7312 /**
7313  * The periodic procedure for triggering asynchronous batch queries for all the
7314  * counter pools. This function is probably called by the host thread.
7315  *
7316  * @param[in] arg
7317  *   The parameter for the alarm process.
7318  */
7319 void
7320 mlx5_flow_query_alarm(void *arg)
7321 {
7322         struct mlx5_dev_ctx_shared *sh = arg;
7323         int ret;
7324         uint16_t pool_index = sh->cmng.pool_index;
7325         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7326         struct mlx5_flow_counter_pool *pool;
7327         uint16_t n_valid;
7328
7329         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7330                 goto set_alarm;
7331         rte_spinlock_lock(&cmng->pool_update_sl);
7332         pool = cmng->pools[pool_index];
7333         n_valid = cmng->n_valid;
7334         rte_spinlock_unlock(&cmng->pool_update_sl);
7335         /* Set the statistic memory to the new created pool. */
7336         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7337                 goto set_alarm;
7338         if (pool->raw_hw)
7339                 /* There is a pool query in progress. */
7340                 goto set_alarm;
7341         pool->raw_hw =
7342                 LIST_FIRST(&sh->cmng.free_stat_raws);
7343         if (!pool->raw_hw)
7344                 /* No free counter statistics raw memory. */
7345                 goto set_alarm;
7346         /*
7347          * Identify the counters released between query trigger and query
7348          * handle more efficiently. The counter released in this gap period
7349          * should wait for a new round of query as the new arrived packets
7350          * will not be taken into account.
7351          */
7352         pool->query_gen++;
7353         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7354                                                MLX5_COUNTERS_PER_POOL,
7355                                                NULL, NULL,
7356                                                pool->raw_hw->mem_mng->dm->id,
7357                                                (void *)(uintptr_t)
7358                                                pool->raw_hw->data,
7359                                                sh->devx_comp,
7360                                                (uint64_t)(uintptr_t)pool);
7361         if (ret) {
7362                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7363                         " %d", pool->min_dcs->id);
7364                 pool->raw_hw = NULL;
7365                 goto set_alarm;
7366         }
7367         LIST_REMOVE(pool->raw_hw, next);
7368         sh->cmng.pending_queries++;
7369         pool_index++;
7370         if (pool_index >= n_valid)
7371                 pool_index = 0;
7372 set_alarm:
7373         sh->cmng.pool_index = pool_index;
7374         mlx5_set_query_alarm(sh);
7375 }
7376
7377 /**
7378  * Check and callback event for new aged flow in the counter pool
7379  *
7380  * @param[in] sh
7381  *   Pointer to mlx5_dev_ctx_shared object.
7382  * @param[in] pool
7383  *   Pointer to Current counter pool.
7384  */
7385 static void
7386 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7387                    struct mlx5_flow_counter_pool *pool)
7388 {
7389         struct mlx5_priv *priv;
7390         struct mlx5_flow_counter *cnt;
7391         struct mlx5_age_info *age_info;
7392         struct mlx5_age_param *age_param;
7393         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7394         struct mlx5_counter_stats_raw *prev = pool->raw;
7395         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7396         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7397         uint16_t expected = AGE_CANDIDATE;
7398         uint32_t i;
7399
7400         pool->time_of_last_age_check = curr_time;
7401         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7402                 cnt = MLX5_POOL_GET_CNT(pool, i);
7403                 age_param = MLX5_CNT_TO_AGE(cnt);
7404                 if (__atomic_load_n(&age_param->state,
7405                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7406                         continue;
7407                 if (cur->data[i].hits != prev->data[i].hits) {
7408                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7409                                          __ATOMIC_RELAXED);
7410                         continue;
7411                 }
7412                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7413                                        time_delta,
7414                                        __ATOMIC_RELAXED) <= age_param->timeout)
7415                         continue;
7416                 /**
7417                  * Hold the lock first, or if between the
7418                  * state AGE_TMOUT and tailq operation the
7419                  * release happened, the release procedure
7420                  * may delete a non-existent tailq node.
7421                  */
7422                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7423                 age_info = GET_PORT_AGE_INFO(priv);
7424                 rte_spinlock_lock(&age_info->aged_sl);
7425                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7426                                                 AGE_TMOUT, false,
7427                                                 __ATOMIC_RELAXED,
7428                                                 __ATOMIC_RELAXED)) {
7429                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7430                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7431                 }
7432                 rte_spinlock_unlock(&age_info->aged_sl);
7433         }
7434         for (i = 0; i < sh->max_port; i++) {
7435                 age_info = &sh->port[i].age_info;
7436                 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW))
7437                         continue;
7438                 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER))
7439                         rte_eth_dev_callback_process
7440                                 (&rte_eth_devices[sh->port[i].devx_ih_port_id],
7441                                 RTE_ETH_EVENT_FLOW_AGED, NULL);
7442                 age_info->flags = 0;
7443         }
7444 }
7445
7446 /**
7447  * Handler for the HW respond about ready values from an asynchronous batch
7448  * query. This function is probably called by the host thread.
7449  *
7450  * @param[in] sh
7451  *   The pointer to the shared device context.
7452  * @param[in] async_id
7453  *   The Devx async ID.
7454  * @param[in] status
7455  *   The status of the completion.
7456  */
7457 void
7458 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7459                                   uint64_t async_id, int status)
7460 {
7461         struct mlx5_flow_counter_pool *pool =
7462                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7463         struct mlx5_counter_stats_raw *raw_to_free;
7464         uint8_t query_gen = pool->query_gen ^ 1;
7465         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7466         enum mlx5_counter_type cnt_type =
7467                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7468                                 MLX5_COUNTER_TYPE_ORIGIN;
7469
7470         if (unlikely(status)) {
7471                 raw_to_free = pool->raw_hw;
7472         } else {
7473                 raw_to_free = pool->raw;
7474                 if (pool->is_aged)
7475                         mlx5_flow_aging_check(sh, pool);
7476                 rte_spinlock_lock(&pool->sl);
7477                 pool->raw = pool->raw_hw;
7478                 rte_spinlock_unlock(&pool->sl);
7479                 /* Be sure the new raw counters data is updated in memory. */
7480                 rte_io_wmb();
7481                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7482                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7483                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7484                                      &pool->counters[query_gen], next);
7485                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7486                 }
7487         }
7488         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7489         pool->raw_hw = NULL;
7490         sh->cmng.pending_queries--;
7491 }
7492
7493 static const struct mlx5_flow_tbl_data_entry  *
7494 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7495 {
7496         struct mlx5_priv *priv = dev->data->dev_private;
7497         struct mlx5_dev_ctx_shared *sh = priv->sh;
7498         struct mlx5_hlist_entry *he;
7499         union tunnel_offload_mark mbits = { .val = mark };
7500         union mlx5_flow_tbl_key table_key = {
7501                 {
7502                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7503                         .reserved = 0,
7504                         .domain = !!mbits.transfer,
7505                         .direction = 0,
7506                 }
7507         };
7508         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
7509         return he ?
7510                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7511 }
7512
7513 static uint32_t
7514 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7515                                 const struct mlx5_flow_tunnel *tunnel,
7516                                 uint32_t group, uint32_t *table,
7517                                 struct rte_flow_error *error)
7518 {
7519         struct mlx5_hlist_entry *he;
7520         struct tunnel_tbl_entry *tte;
7521         union tunnel_tbl_key key = {
7522                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7523                 .group = group
7524         };
7525         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7526         struct mlx5_hlist *group_hash;
7527
7528         group_hash = tunnel ? tunnel->groups : thub->groups;
7529         he = mlx5_hlist_lookup(group_hash, key.val);
7530         if (!he) {
7531                 int ret;
7532                 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7533                                   sizeof(*tte), 0,
7534                                   SOCKET_ID_ANY);
7535                 if (!tte)
7536                         goto err;
7537                 tte->hash.key = key.val;
7538                 ret = mlx5_flow_id_get(thub->table_ids, &tte->flow_table);
7539                 if (ret) {
7540                         mlx5_free(tte);
7541                         goto err;
7542                 }
7543                 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7544                 mlx5_hlist_insert(group_hash, &tte->hash);
7545         } else {
7546                 tte = container_of(he, typeof(*tte), hash);
7547         }
7548         *table = tte->flow_table;
7549         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7550                 dev->data->port_id, key.tunnel_id, group, *table);
7551         return 0;
7552
7553 err:
7554         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7555                                   NULL, "tunnel group index not supported");
7556 }
7557
7558 static int
7559 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7560                     struct flow_grp_info grp_info, struct rte_flow_error *error)
7561 {
7562         if (grp_info.transfer && grp_info.external && grp_info.fdb_def_rule) {
7563                 if (group == UINT32_MAX)
7564                         return rte_flow_error_set
7565                                                 (error, EINVAL,
7566                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7567                                                  NULL,
7568                                                  "group index not supported");
7569                 *table = group + 1;
7570         } else {
7571                 *table = group;
7572         }
7573         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7574         return 0;
7575 }
7576
7577 /**
7578  * Translate the rte_flow group index to HW table value.
7579  *
7580  * If tunnel offload is disabled, all group ids converted to flow table
7581  * id using the standard method.
7582  * If tunnel offload is enabled, group id can be converted using the
7583  * standard or tunnel conversion method. Group conversion method
7584  * selection depends on flags in `grp_info` parameter:
7585  * - Internal (grp_info.external == 0) groups conversion uses the
7586  *   standard method.
7587  * - Group ids in JUMP action converted with the tunnel conversion.
7588  * - Group id in rule attribute conversion depends on a rule type and
7589  *   group id value:
7590  *   ** non zero group attributes converted with the tunnel method
7591  *   ** zero group attribute in non-tunnel rule is converted using the
7592  *      standard method - there's only one root table
7593  *   ** zero group attribute in steer tunnel rule is converted with the
7594  *      standard method - single root table
7595  *   ** zero group attribute in match tunnel rule is a special OvS
7596  *      case: that value is used for portability reasons. That group
7597  *      id is converted with the tunnel conversion method.
7598  *
7599  * @param[in] dev
7600  *   Port device
7601  * @param[in] tunnel
7602  *   PMD tunnel offload object
7603  * @param[in] group
7604  *   rte_flow group index value.
7605  * @param[out] table
7606  *   HW table value.
7607  * @param[in] grp_info
7608  *   flags used for conversion
7609  * @param[out] error
7610  *   Pointer to error structure.
7611  *
7612  * @return
7613  *   0 on success, a negative errno value otherwise and rte_errno is set.
7614  */
7615 int
7616 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7617                          const struct mlx5_flow_tunnel *tunnel,
7618                          uint32_t group, uint32_t *table,
7619                          struct flow_grp_info grp_info,
7620                          struct rte_flow_error *error)
7621 {
7622         int ret;
7623         bool standard_translation;
7624
7625         if (grp_info.external && group < MLX5_MAX_TABLES_EXTERNAL)
7626                 group *= MLX5_FLOW_TABLE_FACTOR;
7627         if (is_tunnel_offload_active(dev)) {
7628                 standard_translation = !grp_info.external ||
7629                                         grp_info.std_tbl_fix;
7630         } else {
7631                 standard_translation = true;
7632         }
7633         DRV_LOG(DEBUG,
7634                 "port %u group=%#x transfer=%d external=%d fdb_def_rule=%d translate=%s",
7635                 dev->data->port_id, group, grp_info.transfer,
7636                 grp_info.external, grp_info.fdb_def_rule,
7637                 standard_translation ? "STANDARD" : "TUNNEL");
7638         if (standard_translation)
7639                 ret = flow_group_to_table(dev->data->port_id, group, table,
7640                                           grp_info, error);
7641         else
7642                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7643                                                       table, error);
7644
7645         return ret;
7646 }
7647
7648 /**
7649  * Discover availability of metadata reg_c's.
7650  *
7651  * Iteratively use test flows to check availability.
7652  *
7653  * @param[in] dev
7654  *   Pointer to the Ethernet device structure.
7655  *
7656  * @return
7657  *   0 on success, a negative errno value otherwise and rte_errno is set.
7658  */
7659 int
7660 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7661 {
7662         struct mlx5_priv *priv = dev->data->dev_private;
7663         struct mlx5_dev_config *config = &priv->config;
7664         enum modify_reg idx;
7665         int n = 0;
7666
7667         /* reg_c[0] and reg_c[1] are reserved. */
7668         config->flow_mreg_c[n++] = REG_C_0;
7669         config->flow_mreg_c[n++] = REG_C_1;
7670         /* Discover availability of other reg_c's. */
7671         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7672                 struct rte_flow_attr attr = {
7673                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7674                         .priority = MLX5_FLOW_PRIO_RSVD,
7675                         .ingress = 1,
7676                 };
7677                 struct rte_flow_item items[] = {
7678                         [0] = {
7679                                 .type = RTE_FLOW_ITEM_TYPE_END,
7680                         },
7681                 };
7682                 struct rte_flow_action actions[] = {
7683                         [0] = {
7684                                 .type = (enum rte_flow_action_type)
7685                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7686                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7687                                         .src = REG_C_1,
7688                                         .dst = idx,
7689                                 },
7690                         },
7691                         [1] = {
7692                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7693                                 .conf = &(struct rte_flow_action_jump){
7694                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7695                                 },
7696                         },
7697                         [2] = {
7698                                 .type = RTE_FLOW_ACTION_TYPE_END,
7699                         },
7700                 };
7701                 uint32_t flow_idx;
7702                 struct rte_flow *flow;
7703                 struct rte_flow_error error;
7704
7705                 if (!config->dv_flow_en)
7706                         break;
7707                 /* Create internal flow, validation skips copy action. */
7708                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7709                                             actions, false, &error);
7710                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7711                                       flow_idx);
7712                 if (!flow)
7713                         continue;
7714                 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
7715                         config->flow_mreg_c[n++] = idx;
7716                 flow_list_destroy(dev, NULL, flow_idx);
7717         }
7718         for (; n < MLX5_MREG_C_NUM; ++n)
7719                 config->flow_mreg_c[n] = REG_NON;
7720         return 0;
7721 }
7722
7723 /**
7724  * Dump flow raw hw data to file
7725  *
7726  * @param[in] dev
7727  *    The pointer to Ethernet device.
7728  * @param[in] file
7729  *   A pointer to a file for output.
7730  * @param[out] error
7731  *   Perform verbose error reporting if not NULL. PMDs initialize this
7732  *   structure in case of error only.
7733  * @return
7734  *   0 on success, a nagative value otherwise.
7735  */
7736 int
7737 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
7738                    FILE *file,
7739                    struct rte_flow_error *error __rte_unused)
7740 {
7741         struct mlx5_priv *priv = dev->data->dev_private;
7742         struct mlx5_dev_ctx_shared *sh = priv->sh;
7743
7744         if (!priv->config.dv_flow_en) {
7745                 if (fputs("device dv flow disabled\n", file) <= 0)
7746                         return -errno;
7747                 return -ENOTSUP;
7748         }
7749         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
7750                                        sh->tx_domain, file);
7751 }
7752
7753 /**
7754  * Get aged-out flows.
7755  *
7756  * @param[in] dev
7757  *   Pointer to the Ethernet device structure.
7758  * @param[in] context
7759  *   The address of an array of pointers to the aged-out flows contexts.
7760  * @param[in] nb_countexts
7761  *   The length of context array pointers.
7762  * @param[out] error
7763  *   Perform verbose error reporting if not NULL. Initialized in case of
7764  *   error only.
7765  *
7766  * @return
7767  *   how many contexts get in success, otherwise negative errno value.
7768  *   if nb_contexts is 0, return the amount of all aged contexts.
7769  *   if nb_contexts is not 0 , return the amount of aged flows reported
7770  *   in the context array.
7771  */
7772 int
7773 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
7774                         uint32_t nb_contexts, struct rte_flow_error *error)
7775 {
7776         const struct mlx5_flow_driver_ops *fops;
7777         struct rte_flow_attr attr = { .transfer = 0 };
7778
7779         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7780                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7781                 return fops->get_aged_flows(dev, contexts, nb_contexts,
7782                                                     error);
7783         }
7784         DRV_LOG(ERR,
7785                 "port %u get aged flows is not supported.",
7786                  dev->data->port_id);
7787         return -ENOTSUP;
7788 }
7789
7790 /* Wrapper for driver action_validate op callback */
7791 static int
7792 flow_drv_action_validate(struct rte_eth_dev *dev,
7793                          const struct rte_flow_shared_action_conf *conf,
7794                          const struct rte_flow_action *action,
7795                          const struct mlx5_flow_driver_ops *fops,
7796                          struct rte_flow_error *error)
7797 {
7798         static const char err_msg[] = "shared action validation unsupported";
7799
7800         if (!fops->action_validate) {
7801                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7802                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7803                                    NULL, err_msg);
7804                 return -rte_errno;
7805         }
7806         return fops->action_validate(dev, conf, action, error);
7807 }
7808
7809 /**
7810  * Destroys the shared action by handle.
7811  *
7812  * @param dev
7813  *   Pointer to Ethernet device structure.
7814  * @param[in] action
7815  *   Handle for the shared action to be destroyed.
7816  * @param[out] error
7817  *   Perform verbose error reporting if not NULL. PMDs initialize this
7818  *   structure in case of error only.
7819  *
7820  * @return
7821  *   0 on success, a negative errno value otherwise and rte_errno is set.
7822  *
7823  * @note: wrapper for driver action_create op callback.
7824  */
7825 static int
7826 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
7827                            struct rte_flow_shared_action *action,
7828                            struct rte_flow_error *error)
7829 {
7830         static const char err_msg[] = "shared action destruction unsupported";
7831         struct rte_flow_attr attr = { .transfer = 0 };
7832         const struct mlx5_flow_driver_ops *fops =
7833                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7834
7835         if (!fops->action_destroy) {
7836                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7837                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7838                                    NULL, err_msg);
7839                 return -rte_errno;
7840         }
7841         return fops->action_destroy(dev, action, error);
7842 }
7843
7844 /* Wrapper for driver action_destroy op callback */
7845 static int
7846 flow_drv_action_update(struct rte_eth_dev *dev,
7847                        struct rte_flow_shared_action *action,
7848                        const void *action_conf,
7849                        const struct mlx5_flow_driver_ops *fops,
7850                        struct rte_flow_error *error)
7851 {
7852         static const char err_msg[] = "shared action update unsupported";
7853
7854         if (!fops->action_update) {
7855                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7856                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7857                                    NULL, err_msg);
7858                 return -rte_errno;
7859         }
7860         return fops->action_update(dev, action, action_conf, error);
7861 }
7862
7863 /**
7864  * Create shared action for reuse in multiple flow rules.
7865  *
7866  * @param dev
7867  *   Pointer to Ethernet device structure.
7868  * @param[in] action
7869  *   Action configuration for shared action creation.
7870  * @param[out] error
7871  *   Perform verbose error reporting if not NULL. PMDs initialize this
7872  *   structure in case of error only.
7873  * @return
7874  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7875  */
7876 static struct rte_flow_shared_action *
7877 mlx5_shared_action_create(struct rte_eth_dev *dev,
7878                           const struct rte_flow_shared_action_conf *conf,
7879                           const struct rte_flow_action *action,
7880                           struct rte_flow_error *error)
7881 {
7882         static const char err_msg[] = "shared action creation unsupported";
7883         struct rte_flow_attr attr = { .transfer = 0 };
7884         const struct mlx5_flow_driver_ops *fops =
7885                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7886
7887         if (flow_drv_action_validate(dev, conf, action, fops, error))
7888                 return NULL;
7889         if (!fops->action_create) {
7890                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7891                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7892                                    NULL, err_msg);
7893                 return NULL;
7894         }
7895         return fops->action_create(dev, conf, action, error);
7896 }
7897
7898 /**
7899  * Updates inplace the shared action configuration pointed by *action* handle
7900  * with the configuration provided as *action* argument.
7901  * The update of the shared action configuration effects all flow rules reusing
7902  * the action via handle.
7903  *
7904  * @param dev
7905  *   Pointer to Ethernet device structure.
7906  * @param[in] shared_action
7907  *   Handle for the shared action to be updated.
7908  * @param[in] action
7909  *   Action specification used to modify the action pointed by handle.
7910  *   *action* should be of same type with the action pointed by the *action*
7911  *   handle argument, otherwise considered as invalid.
7912  * @param[out] error
7913  *   Perform verbose error reporting if not NULL. PMDs initialize this
7914  *   structure in case of error only.
7915  *
7916  * @return
7917  *   0 on success, a negative errno value otherwise and rte_errno is set.
7918  */
7919 static int
7920 mlx5_shared_action_update(struct rte_eth_dev *dev,
7921                 struct rte_flow_shared_action *shared_action,
7922                 const struct rte_flow_action *action,
7923                 struct rte_flow_error *error)
7924 {
7925         struct rte_flow_attr attr = { .transfer = 0 };
7926         const struct mlx5_flow_driver_ops *fops =
7927                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7928         int ret;
7929
7930         switch (shared_action->type) {
7931         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
7932                 if (action->type != RTE_FLOW_ACTION_TYPE_RSS) {
7933                         return rte_flow_error_set(error, EINVAL,
7934                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7935                                                   NULL,
7936                                                   "update action type invalid");
7937                 }
7938                 ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7939                 if (ret)
7940                         return ret;
7941                 return flow_drv_action_update(dev, shared_action, action->conf,
7942                                               fops, error);
7943         default:
7944                 return rte_flow_error_set(error, ENOTSUP,
7945                                           RTE_FLOW_ERROR_TYPE_ACTION,
7946                                           NULL,
7947                                           "action type not supported");
7948         }
7949 }
7950
7951 /**
7952  * Query the shared action by handle.
7953  *
7954  * This function allows retrieving action-specific data such as counters.
7955  * Data is gathered by special action which may be present/referenced in
7956  * more than one flow rule definition.
7957  *
7958  * \see RTE_FLOW_ACTION_TYPE_COUNT
7959  *
7960  * @param dev
7961  *   Pointer to Ethernet device structure.
7962  * @param[in] action
7963  *   Handle for the shared action to query.
7964  * @param[in, out] data
7965  *   Pointer to storage for the associated query data type.
7966  * @param[out] error
7967  *   Perform verbose error reporting if not NULL. PMDs initialize this
7968  *   structure in case of error only.
7969  *
7970  * @return
7971  *   0 on success, a negative errno value otherwise and rte_errno is set.
7972  */
7973 static int
7974 mlx5_shared_action_query(struct rte_eth_dev *dev,
7975                          const struct rte_flow_shared_action *action,
7976                          void *data,
7977                          struct rte_flow_error *error)
7978 {
7979         (void)dev;
7980         switch (action->type) {
7981         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
7982                 __atomic_load(&action->refcnt, (uint32_t *)data,
7983                               __ATOMIC_RELAXED);
7984                 return 0;
7985         default:
7986                 return rte_flow_error_set(error, ENOTSUP,
7987                                           RTE_FLOW_ERROR_TYPE_ACTION,
7988                                           NULL,
7989                                           "action type not supported");
7990         }
7991 }
7992
7993 /**
7994  * Destroy all shared actions.
7995  *
7996  * @param dev
7997  *   Pointer to Ethernet device.
7998  *
7999  * @return
8000  *   0 on success, a negative errno value otherwise and rte_errno is set.
8001  */
8002 int
8003 mlx5_shared_action_flush(struct rte_eth_dev *dev)
8004 {
8005         struct rte_flow_error error;
8006         struct mlx5_priv *priv = dev->data->dev_private;
8007         struct rte_flow_shared_action *action;
8008         int ret = 0;
8009
8010         while (!LIST_EMPTY(&priv->shared_actions)) {
8011                 action = LIST_FIRST(&priv->shared_actions);
8012                 ret = mlx5_shared_action_destroy(dev, action, &error);
8013         }
8014         return ret;
8015 }
8016
8017 static void
8018 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8019                       struct mlx5_flow_tunnel *tunnel)
8020 {
8021         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8022         struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids;
8023
8024         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8025                 dev->data->port_id, tunnel->tunnel_id);
8026         RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED));
8027         LIST_REMOVE(tunnel, chain);
8028         mlx5_flow_id_release(id_pool, tunnel->tunnel_id);
8029         mlx5_hlist_destroy(tunnel->groups, NULL, NULL);
8030         mlx5_free(tunnel);
8031 }
8032
8033 static struct mlx5_flow_tunnel *
8034 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8035 {
8036         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8037         struct mlx5_flow_tunnel *tun;
8038
8039         LIST_FOREACH(tun, &thub->tunnels, chain) {
8040                 if (tun->tunnel_id == id)
8041                         break;
8042         }
8043
8044         return tun;
8045 }
8046
8047 static struct mlx5_flow_tunnel *
8048 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8049                           const struct rte_flow_tunnel *app_tunnel)
8050 {
8051         int ret;
8052         struct mlx5_flow_tunnel *tunnel;
8053         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8054         struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids;
8055         uint32_t id;
8056
8057         ret = mlx5_flow_id_get(id_pool, &id);
8058         if (ret)
8059                 return NULL;
8060         /**
8061          * mlx5 flow tunnel is an auxlilary data structure
8062          * It's not part of IO. No need to allocate it from
8063          * huge pages pools dedicated for IO
8064          */
8065         tunnel = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*tunnel),
8066                              0, SOCKET_ID_ANY);
8067         if (!tunnel) {
8068                 mlx5_flow_id_pool_release(id_pool);
8069                 return NULL;
8070         }
8071         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024);
8072         if (!tunnel->groups) {
8073                 mlx5_flow_id_pool_release(id_pool);
8074                 mlx5_free(tunnel);
8075                 return NULL;
8076         }
8077         /* initiate new PMD tunnel */
8078         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8079         tunnel->tunnel_id = id;
8080         tunnel->action.type = (typeof(tunnel->action.type))
8081                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8082         tunnel->action.conf = tunnel;
8083         tunnel->item.type = (typeof(tunnel->item.type))
8084                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8085         tunnel->item.spec = tunnel;
8086         tunnel->item.last = NULL;
8087         tunnel->item.mask = NULL;
8088
8089         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8090                 dev->data->port_id, tunnel->tunnel_id);
8091
8092         return tunnel;
8093 }
8094
8095 static int
8096 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8097                      const struct rte_flow_tunnel *app_tunnel,
8098                      struct mlx5_flow_tunnel **tunnel)
8099 {
8100         int ret;
8101         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8102         struct mlx5_flow_tunnel *tun;
8103
8104         LIST_FOREACH(tun, &thub->tunnels, chain) {
8105                 if (!memcmp(app_tunnel, &tun->app_tunnel,
8106                             sizeof(*app_tunnel))) {
8107                         *tunnel = tun;
8108                         ret = 0;
8109                         break;
8110                 }
8111         }
8112         if (!tun) {
8113                 tun = mlx5_flow_tunnel_allocate(dev, app_tunnel);
8114                 if (tun) {
8115                         LIST_INSERT_HEAD(&thub->tunnels, tun, chain);
8116                         *tunnel = tun;
8117                 } else {
8118                         ret = -ENOMEM;
8119                 }
8120         }
8121         if (tun)
8122                 __atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
8123
8124         return ret;
8125 }
8126
8127 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8128 {
8129         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8130
8131         if (!thub)
8132                 return;
8133         if (!LIST_EMPTY(&thub->tunnels))
8134                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
8135         mlx5_flow_id_pool_release(thub->tunnel_ids);
8136         mlx5_flow_id_pool_release(thub->table_ids);
8137         mlx5_hlist_destroy(thub->groups, NULL, NULL);
8138         mlx5_free(thub);
8139 }
8140
8141 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8142 {
8143         int err;
8144         struct mlx5_flow_tunnel_hub *thub;
8145
8146         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8147                            0, SOCKET_ID_ANY);
8148         if (!thub)
8149                 return -ENOMEM;
8150         LIST_INIT(&thub->tunnels);
8151         thub->tunnel_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TUNNELS);
8152         if (!thub->tunnel_ids) {
8153                 err = -rte_errno;
8154                 goto err;
8155         }
8156         thub->table_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TABLES);
8157         if (!thub->table_ids) {
8158                 err = -rte_errno;
8159                 goto err;
8160         }
8161         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES);
8162         if (!thub->groups) {
8163                 err = -rte_errno;
8164                 goto err;
8165         }
8166         sh->tunnel_hub = thub;
8167
8168         return 0;
8169
8170 err:
8171         if (thub->groups)
8172                 mlx5_hlist_destroy(thub->groups, NULL, NULL);
8173         if (thub->table_ids)
8174                 mlx5_flow_id_pool_release(thub->table_ids);
8175         if (thub->tunnel_ids)
8176                 mlx5_flow_id_pool_release(thub->tunnel_ids);
8177         if (thub)
8178                 mlx5_free(thub);
8179         return err;
8180 }
8181
8182 #ifndef HAVE_MLX5DV_DR
8183 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8184 #else
8185 #define MLX5_DOMAIN_SYNC_FLOW \
8186         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8187 #endif
8188
8189 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8190 {
8191         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8192         const struct mlx5_flow_driver_ops *fops;
8193         int ret;
8194         struct rte_flow_attr attr = { .transfer = 0 };
8195
8196         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8197         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8198         if (ret > 0)
8199                 ret = -ret;
8200         return ret;
8201 }