drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_eal_paging.h>
  17 #include <rte_flow.h>
  18 #include <rte_cycles.h>
  19 #include <rte_flow_driver.h>
  20 #include <rte_malloc.h>
  21 #include <rte_ip.h>
  22
  23 #include <mlx5_glue.h>
  24 #include <mlx5_devx_cmds.h>
  25 #include <mlx5_prm.h>
  26 #include <mlx5_malloc.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_flow.h"
  31 #include "mlx5_flow_os.h"
  32 #include "mlx5_rxtx.h"
  33 #include "mlx5_common_os.h"
  34
  35 static struct mlx5_flow_tunnel *
  36 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  37 static void
  38 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  39 static const struct mlx5_flow_tbl_data_entry  *
  40 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark);
  41 static int
  42 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
  43                      const struct rte_flow_tunnel *app_tunnel,
  44                      struct mlx5_flow_tunnel **tunnel);
  45
  46
  47 /** Device flow drivers. */
  48 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  49
  50 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  51
  52 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  53         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  54 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  55         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  56 #endif
  57         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  58         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  59 };
  60
  61 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  62 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  63         (const int []){ \
  64                 __VA_ARGS__, 0, \
  65         }
  66
  67 /** Node object of input graph for mlx5_flow_expand_rss(). */
  68 struct mlx5_flow_expand_node {
  69         const int *const next;
  70         /**<
  71          * List of next node indexes. Index 0 is interpreted as a terminator.
  72          */
  73         const enum rte_flow_item_type type;
  74         /**< Pattern item type of current node. */
  75         uint64_t rss_types;
  76         /**<
  77          * RSS types bit-field associated with this node
  78          * (see ETH_RSS_* definitions).
  79          */
  80 };
  81
  82 /** Object returned by mlx5_flow_expand_rss(). */
  83 struct mlx5_flow_expand_rss {
  84         uint32_t entries;
  85         /**< Number of entries @p patterns and @p priorities. */
  86         struct {
  87                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
  88                 uint32_t priority; /**< Priority offset for each expansion. */
  89         } entry[];
  90 };
  91
  92 static enum rte_flow_item_type
  93 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
  94 {
  95         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
  96         uint16_t ether_type = 0;
  97         uint16_t ether_type_m;
  98         uint8_t ip_next_proto = 0;
  99         uint8_t ip_next_proto_m;
 100
 101         if (item == NULL || item->spec == NULL)
 102                 return ret;
 103         switch (item->type) {
 104         case RTE_FLOW_ITEM_TYPE_ETH:
 105                 if (item->mask)
 106                         ether_type_m = ((const struct rte_flow_item_eth *)
 107                                                 (item->mask))->type;
 108                 else
 109                         ether_type_m = rte_flow_item_eth_mask.type;
 110                 if (ether_type_m != RTE_BE16(0xFFFF))
 111                         break;
 112                 ether_type = ((const struct rte_flow_item_eth *)
 113                                 (item->spec))->type;
 114                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 115                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 116                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 117                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 118                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 119                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 120                 else
 121                         ret = RTE_FLOW_ITEM_TYPE_END;
 122                 break;
 123         case RTE_FLOW_ITEM_TYPE_VLAN:
 124                 if (item->mask)
 125                         ether_type_m = ((const struct rte_flow_item_vlan *)
 126                                                 (item->mask))->inner_type;
 127                 else
 128                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 129                 if (ether_type_m != RTE_BE16(0xFFFF))
 130                         break;
 131                 ether_type = ((const struct rte_flow_item_vlan *)
 132                                 (item->spec))->inner_type;
 133                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 134                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 135                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 136                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 137                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 138                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 139                 else
 140                         ret = RTE_FLOW_ITEM_TYPE_END;
 141                 break;
 142         case RTE_FLOW_ITEM_TYPE_IPV4:
 143                 if (item->mask)
 144                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 145                                         (item->mask))->hdr.next_proto_id;
 146                 else
 147                         ip_next_proto_m =
 148                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 149                 if (ip_next_proto_m != 0xFF)
 150                         break;
 151                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 152                                 (item->spec))->hdr.next_proto_id;
 153                 if (ip_next_proto == IPPROTO_UDP)
 154                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 155                 else if (ip_next_proto == IPPROTO_TCP)
 156                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 157                 else if (ip_next_proto == IPPROTO_IP)
 158                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 159                 else if (ip_next_proto == IPPROTO_IPV6)
 160                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 161                 else
 162                         ret = RTE_FLOW_ITEM_TYPE_END;
 163                 break;
 164         case RTE_FLOW_ITEM_TYPE_IPV6:
 165                 if (item->mask)
 166                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 167                                                 (item->mask))->hdr.proto;
 168                 else
 169                         ip_next_proto_m =
 170                                 rte_flow_item_ipv6_mask.hdr.proto;
 171                 if (ip_next_proto_m != 0xFF)
 172                         break;
 173                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 174                                 (item->spec))->hdr.proto;
 175                 if (ip_next_proto == IPPROTO_UDP)
 176                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 177                 else if (ip_next_proto == IPPROTO_TCP)
 178                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 179                 else if (ip_next_proto == IPPROTO_IP)
 180                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 181                 else if (ip_next_proto == IPPROTO_IPV6)
 182                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 183                 else
 184                         ret = RTE_FLOW_ITEM_TYPE_END;
 185                 break;
 186         default:
 187                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 188                 break;
 189         }
 190         return ret;
 191 }
 192
 193 /**
 194  * Expand RSS flows into several possible flows according to the RSS hash
 195  * fields requested and the driver capabilities.
 196  *
 197  * @param[out] buf
 198  *   Buffer to store the result expansion.
 199  * @param[in] size
 200  *   Buffer size in bytes. If 0, @p buf can be NULL.
 201  * @param[in] pattern
 202  *   User flow pattern.
 203  * @param[in] types
 204  *   RSS types to expand (see ETH_RSS_* definitions).
 205  * @param[in] graph
 206  *   Input graph to expand @p pattern according to @p types.
 207  * @param[in] graph_root_index
 208  *   Index of root node in @p graph, typically 0.
 209  *
 210  * @return
 211  *   A positive value representing the size of @p buf in bytes regardless of
 212  *   @p size on success, a negative errno value otherwise and rte_errno is
 213  *   set, the following errors are defined:
 214  *
 215  *   -E2BIG: graph-depth @p graph is too deep.
 216  */
 217 static int
 218 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 219                      const struct rte_flow_item *pattern, uint64_t types,
 220                      const struct mlx5_flow_expand_node graph[],
 221                      int graph_root_index)
 222 {
 223         const int elt_n = 8;
 224         const struct rte_flow_item *item;
 225         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 226         const int *next_node;
 227         const int *stack[elt_n];
 228         int stack_pos = 0;
 229         struct rte_flow_item flow_items[elt_n];
 230         unsigned int i;
 231         size_t lsize;
 232         size_t user_pattern_size = 0;
 233         void *addr = NULL;
 234         const struct mlx5_flow_expand_node *next = NULL;
 235         struct rte_flow_item missed_item;
 236         int missed = 0;
 237         int elt = 0;
 238         const struct rte_flow_item *last_item = NULL;
 239
 240         memset(&missed_item, 0, sizeof(missed_item));
 241         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 242                 elt_n * sizeof(buf->entry[0]);
 243         if (lsize <= size) {
 244                 buf->entry[0].priority = 0;
 245                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 246                 buf->entries = 0;
 247                 addr = buf->entry[0].pattern;
 248         }
 249         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 250                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 251                         last_item = item;
 252                 for (i = 0; node->next && node->next[i]; ++i) {
 253                         next = &graph[node->next[i]];
 254                         if (next->type == item->type)
 255                                 break;
 256                 }
 257                 if (next)
 258                         node = next;
 259                 user_pattern_size += sizeof(*item);
 260         }
 261         user_pattern_size += sizeof(*item); /* Handle END item. */
 262         lsize += user_pattern_size;
 263         /* Copy the user pattern in the first entry of the buffer. */
 264         if (lsize <= size) {
 265                 rte_memcpy(addr, pattern, user_pattern_size);
 266                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 267                 buf->entries = 1;
 268         }
 269         /* Start expanding. */
 270         memset(flow_items, 0, sizeof(flow_items));
 271         user_pattern_size -= sizeof(*item);
 272         /*
 273          * Check if the last valid item has spec set, need complete pattern,
 274          * and the pattern can be used for expansion.
 275          */
 276         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 277         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 278                 /* Item type END indicates expansion is not required. */
 279                 return lsize;
 280         }
 281         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 282                 next = NULL;
 283                 missed = 1;
 284                 for (i = 0; node->next && node->next[i]; ++i) {
 285                         next = &graph[node->next[i]];
 286                         if (next->type == missed_item.type) {
 287                                 flow_items[0].type = missed_item.type;
 288                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 289                                 break;
 290                         }
 291                         next = NULL;
 292                 }
 293         }
 294         if (next && missed) {
 295                 elt = 2; /* missed item + item end. */
 296                 node = next;
 297                 lsize += elt * sizeof(*item) + user_pattern_size;
 298                 if ((node->rss_types & types) && lsize <= size) {
 299                         buf->entry[buf->entries].priority = 1;
 300                         buf->entry[buf->entries].pattern = addr;
 301                         buf->entries++;
 302                         rte_memcpy(addr, buf->entry[0].pattern,
 303                                    user_pattern_size);
 304                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 305                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 306                         addr = (void *)(((uintptr_t)addr) +
 307                                         elt * sizeof(*item));
 308                 }
 309         }
 310         memset(flow_items, 0, sizeof(flow_items));
 311         next_node = node->next;
 312         stack[stack_pos] = next_node;
 313         node = next_node ? &graph[*next_node] : NULL;
 314         while (node) {
 315                 flow_items[stack_pos].type = node->type;
 316                 if (node->rss_types & types) {
 317                         /*
 318                          * compute the number of items to copy from the
 319                          * expansion and copy it.
 320                          * When the stack_pos is 0, there are 1 element in it,
 321                          * plus the addition END item.
 322                          */
 323                         elt = stack_pos + 2;
 324                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 325                         lsize += elt * sizeof(*item) + user_pattern_size;
 326                         if (lsize <= size) {
 327                                 size_t n = elt * sizeof(*item);
 328
 329                                 buf->entry[buf->entries].priority =
 330                                         stack_pos + 1 + missed;
 331                                 buf->entry[buf->entries].pattern = addr;
 332                                 buf->entries++;
 333                                 rte_memcpy(addr, buf->entry[0].pattern,
 334                                            user_pattern_size);
 335                                 addr = (void *)(((uintptr_t)addr) +
 336                                                 user_pattern_size);
 337                                 rte_memcpy(addr, &missed_item,
 338                                            missed * sizeof(*item));
 339                                 addr = (void *)(((uintptr_t)addr) +
 340                                         missed * sizeof(*item));
 341                                 rte_memcpy(addr, flow_items, n);
 342                                 addr = (void *)(((uintptr_t)addr) + n);
 343                         }
 344                 }
 345                 /* Go deeper. */
 346                 if (node->next) {
 347                         next_node = node->next;
 348                         if (stack_pos++ == elt_n) {
 349                                 rte_errno = E2BIG;
 350                                 return -rte_errno;
 351                         }
 352                         stack[stack_pos] = next_node;
 353                 } else if (*(next_node + 1)) {
 354                         /* Follow up with the next possibility. */
 355                         ++next_node;
 356                 } else {
 357                         /* Move to the next path. */
 358                         if (stack_pos)
 359                                 next_node = stack[--stack_pos];
 360                         next_node++;
 361                         stack[stack_pos] = next_node;
 362                 }
 363                 node = *next_node ? &graph[*next_node] : NULL;
 364         };
 365         /* no expanded flows but we have missed item, create one rule for it */
 366         if (buf->entries == 1 && missed != 0) {
 367                 elt = 2;
 368                 lsize += elt * sizeof(*item) + user_pattern_size;
 369                 if (lsize <= size) {
 370                         buf->entry[buf->entries].priority = 1;
 371                         buf->entry[buf->entries].pattern = addr;
 372                         buf->entries++;
 373                         flow_items[0].type = missed_item.type;
 374                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 375                         rte_memcpy(addr, buf->entry[0].pattern,
 376                                    user_pattern_size);
 377                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 378                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 379                         addr = (void *)(((uintptr_t)addr) +
 380                                         elt * sizeof(*item));
 381                 }
 382         }
 383         return lsize;
 384 }
 385
 386 enum mlx5_expansion {
 387         MLX5_EXPANSION_ROOT,
 388         MLX5_EXPANSION_ROOT_OUTER,
 389         MLX5_EXPANSION_ROOT_ETH_VLAN,
 390         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 391         MLX5_EXPANSION_OUTER_ETH,
 392         MLX5_EXPANSION_OUTER_ETH_VLAN,
 393         MLX5_EXPANSION_OUTER_VLAN,
 394         MLX5_EXPANSION_OUTER_IPV4,
 395         MLX5_EXPANSION_OUTER_IPV4_UDP,
 396         MLX5_EXPANSION_OUTER_IPV4_TCP,
 397         MLX5_EXPANSION_OUTER_IPV6,
 398         MLX5_EXPANSION_OUTER_IPV6_UDP,
 399         MLX5_EXPANSION_OUTER_IPV6_TCP,
 400         MLX5_EXPANSION_VXLAN,
 401         MLX5_EXPANSION_VXLAN_GPE,
 402         MLX5_EXPANSION_GRE,
 403         MLX5_EXPANSION_MPLS,
 404         MLX5_EXPANSION_ETH,
 405         MLX5_EXPANSION_ETH_VLAN,
 406         MLX5_EXPANSION_VLAN,
 407         MLX5_EXPANSION_IPV4,
 408         MLX5_EXPANSION_IPV4_UDP,
 409         MLX5_EXPANSION_IPV4_TCP,
 410         MLX5_EXPANSION_IPV6,
 411         MLX5_EXPANSION_IPV6_UDP,
 412         MLX5_EXPANSION_IPV6_TCP,
 413 };
 414
 415 /** Supported expansion of items. */
 416 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 417         [MLX5_EXPANSION_ROOT] = {
 418                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 419                                                   MLX5_EXPANSION_IPV4,
 420                                                   MLX5_EXPANSION_IPV6),
 421                 .type = RTE_FLOW_ITEM_TYPE_END,
 422         },
 423         [MLX5_EXPANSION_ROOT_OUTER] = {
 424                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 425                                                   MLX5_EXPANSION_OUTER_IPV4,
 426                                                   MLX5_EXPANSION_OUTER_IPV6),
 427                 .type = RTE_FLOW_ITEM_TYPE_END,
 428         },
 429         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 430                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 431                 .type = RTE_FLOW_ITEM_TYPE_END,
 432         },
 433         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 434                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 435                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 436                 .type = RTE_FLOW_ITEM_TYPE_END,
 437         },
 438         [MLX5_EXPANSION_OUTER_ETH] = {
 439                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 440                                                   MLX5_EXPANSION_OUTER_IPV6,
 441                                                   MLX5_EXPANSION_MPLS),
 442                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 443                 .rss_types = 0,
 444         },
 445         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 446                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 447                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 448                 .rss_types = 0,
 449         },
 450         [MLX5_EXPANSION_OUTER_VLAN] = {
 451                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 452                                                   MLX5_EXPANSION_OUTER_IPV6),
 453                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 454         },
 455         [MLX5_EXPANSION_OUTER_IPV4] = {
 456                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 457                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 458                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 459                          MLX5_EXPANSION_GRE,
 460                          MLX5_EXPANSION_IPV4,
 461                          MLX5_EXPANSION_IPV6),
 462                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 463                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 464                         ETH_RSS_NONFRAG_IPV4_OTHER,
 465         },
 466         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 467                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 468                                                   MLX5_EXPANSION_VXLAN_GPE),
 469                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 470                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 471         },
 472         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 473                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 474                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 475         },
 476         [MLX5_EXPANSION_OUTER_IPV6] = {
 477                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 478                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 479                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 480                          MLX5_EXPANSION_IPV4,
 481                          MLX5_EXPANSION_IPV6),
 482                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 483                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 484                         ETH_RSS_NONFRAG_IPV6_OTHER,
 485         },
 486         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 487                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 488                                                   MLX5_EXPANSION_VXLAN_GPE),
 489                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 490                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 493                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 494                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 495         },
 496         [MLX5_EXPANSION_VXLAN] = {
 497                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 498                                                   MLX5_EXPANSION_IPV4,
 499                                                   MLX5_EXPANSION_IPV6),
 500                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 501         },
 502         [MLX5_EXPANSION_VXLAN_GPE] = {
 503                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 504                                                   MLX5_EXPANSION_IPV4,
 505                                                   MLX5_EXPANSION_IPV6),
 506                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 507         },
 508         [MLX5_EXPANSION_GRE] = {
 509                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 510                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 511         },
 512         [MLX5_EXPANSION_MPLS] = {
 513                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 514                                                   MLX5_EXPANSION_IPV6),
 515                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 516         },
 517         [MLX5_EXPANSION_ETH] = {
 518                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 519                                                   MLX5_EXPANSION_IPV6),
 520                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 521         },
 522         [MLX5_EXPANSION_ETH_VLAN] = {
 523                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 524                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 525         },
 526         [MLX5_EXPANSION_VLAN] = {
 527                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 528                                                   MLX5_EXPANSION_IPV6),
 529                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 530         },
 531         [MLX5_EXPANSION_IPV4] = {
 532                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 533                                                   MLX5_EXPANSION_IPV4_TCP),
 534                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 535                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 536                         ETH_RSS_NONFRAG_IPV4_OTHER,
 537         },
 538         [MLX5_EXPANSION_IPV4_UDP] = {
 539                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 540                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 541         },
 542         [MLX5_EXPANSION_IPV4_TCP] = {
 543                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 544                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 545         },
 546         [MLX5_EXPANSION_IPV6] = {
 547                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 548                                                   MLX5_EXPANSION_IPV6_TCP),
 549                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 550                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 551                         ETH_RSS_NONFRAG_IPV6_OTHER,
 552         },
 553         [MLX5_EXPANSION_IPV6_UDP] = {
 554                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 555                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 556         },
 557         [MLX5_EXPANSION_IPV6_TCP] = {
 558                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 559                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 560         },
 561 };
 562
 563 static struct rte_flow_shared_action *
 564 mlx5_shared_action_create(struct rte_eth_dev *dev,
 565                           const struct rte_flow_shared_action_conf *conf,
 566                           const struct rte_flow_action *action,
 567                           struct rte_flow_error *error);
 568 static int mlx5_shared_action_destroy
 569                                 (struct rte_eth_dev *dev,
 570                                  struct rte_flow_shared_action *shared_action,
 571                                  struct rte_flow_error *error);
 572 static int mlx5_shared_action_update
 573                                 (struct rte_eth_dev *dev,
 574                                  struct rte_flow_shared_action *shared_action,
 575                                  const struct rte_flow_action *action,
 576                                  struct rte_flow_error *error);
 577 static int mlx5_shared_action_query
 578                                 (struct rte_eth_dev *dev,
 579                                  const struct rte_flow_shared_action *action,
 580                                  void *data,
 581                                  struct rte_flow_error *error);
 582 static inline bool
 583 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
 584                           struct rte_flow_tunnel *tunnel,
 585                           const char *err_msg)
 586 {
 587         err_msg = NULL;
 588         if (!is_tunnel_offload_active(dev)) {
 589                 err_msg = "tunnel offload was not activated";
 590                 goto out;
 591         } else if (!tunnel) {
 592                 err_msg = "no application tunnel";
 593                 goto out;
 594         }
 595
 596         switch (tunnel->type) {
 597         default:
 598                 err_msg = "unsupported tunnel type";
 599                 goto out;
 600         case RTE_FLOW_ITEM_TYPE_VXLAN:
 601                 break;
 602         }
 603
 604 out:
 605         return !err_msg;
 606 }
 607
 608
 609 static int
 610 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 611                     struct rte_flow_tunnel *app_tunnel,
 612                     struct rte_flow_action **actions,
 613                     uint32_t *num_of_actions,
 614                     struct rte_flow_error *error)
 615 {
 616         int ret;
 617         struct mlx5_flow_tunnel *tunnel;
 618         const char *err_msg = NULL;
 619         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
 620
 621         if (!verdict)
 622                 return rte_flow_error_set(error, EINVAL,
 623                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
 624                                           err_msg);
 625         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
 626         if (ret < 0) {
 627                 return rte_flow_error_set(error, ret,
 628                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
 629                                           "failed to initialize pmd tunnel");
 630         }
 631         *actions = &tunnel->action;
 632         *num_of_actions = 1;
 633         return 0;
 634 }
 635
 636 static int
 637 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 638                        struct rte_flow_tunnel *app_tunnel,
 639                        struct rte_flow_item **items,
 640                        uint32_t *num_of_items,
 641                        struct rte_flow_error *error)
 642 {
 643         int ret;
 644         struct mlx5_flow_tunnel *tunnel;
 645         const char *err_msg = NULL;
 646         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
 647
 648         if (!verdict)
 649                 return rte_flow_error_set(error, EINVAL,
 650                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 651                                           err_msg);
 652         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
 653         if (ret < 0) {
 654                 return rte_flow_error_set(error, ret,
 655                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 656                                           "failed to initialize pmd tunnel");
 657         }
 658         *items = &tunnel->item;
 659         *num_of_items = 1;
 660         return 0;
 661 }
 662
 663 static int
 664 mlx5_flow_item_release(struct rte_eth_dev *dev,
 665                        struct rte_flow_item *pmd_items,
 666                        uint32_t num_items, struct rte_flow_error *err)
 667 {
 668         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 669         struct mlx5_flow_tunnel *tun;
 670
 671         LIST_FOREACH(tun, &thub->tunnels, chain) {
 672                 if (&tun->item == pmd_items)
 673                         break;
 674         }
 675         if (!tun || num_items != 1)
 676                 return rte_flow_error_set(err, EINVAL,
 677                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 678                                           "invalid argument");
 679         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
 680                 mlx5_flow_tunnel_free(dev, tun);
 681         return 0;
 682 }
 683
 684 static int
 685 mlx5_flow_action_release(struct rte_eth_dev *dev,
 686                          struct rte_flow_action *pmd_actions,
 687                          uint32_t num_actions, struct rte_flow_error *err)
 688 {
 689         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 690         struct mlx5_flow_tunnel *tun;
 691
 692         LIST_FOREACH(tun, &thub->tunnels, chain) {
 693                 if (&tun->action == pmd_actions)
 694                         break;
 695         }
 696         if (!tun || num_actions != 1)
 697                 return rte_flow_error_set(err, EINVAL,
 698                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 699                                           "invalid argument");
 700         if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED))
 701                 mlx5_flow_tunnel_free(dev, tun);
 702
 703         return 0;
 704 }
 705
 706 static int
 707 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 708                                   struct rte_mbuf *m,
 709                                   struct rte_flow_restore_info *info,
 710                                   struct rte_flow_error *err)
 711 {
 712         uint64_t ol_flags = m->ol_flags;
 713         const struct mlx5_flow_tbl_data_entry *tble;
 714         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
 715
 716         if ((ol_flags & mask) != mask)
 717                 goto err;
 718         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
 719         if (!tble) {
 720                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
 721                         dev->data->port_id, m->hash.fdir.hi);
 722                 goto err;
 723         }
 724         MLX5_ASSERT(tble->tunnel);
 725         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
 726         info->group_id = tble->group_id;
 727         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
 728                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
 729                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
 730
 731         return 0;
 732
 733 err:
 734         return rte_flow_error_set(err, EINVAL,
 735                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 736                                   "failed to get restore info");
 737 }
 738
 739 static const struct rte_flow_ops mlx5_flow_ops = {
 740         .validate = mlx5_flow_validate,
 741         .create = mlx5_flow_create,
 742         .destroy = mlx5_flow_destroy,
 743         .flush = mlx5_flow_flush,
 744         .isolate = mlx5_flow_isolate,
 745         .query = mlx5_flow_query,
 746         .dev_dump = mlx5_flow_dev_dump,
 747         .get_aged_flows = mlx5_flow_get_aged_flows,
 748         .shared_action_create = mlx5_shared_action_create,
 749         .shared_action_destroy = mlx5_shared_action_destroy,
 750         .shared_action_update = mlx5_shared_action_update,
 751         .shared_action_query = mlx5_shared_action_query,
 752         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 753         .tunnel_match = mlx5_flow_tunnel_match,
 754         .tunnel_action_decap_release = mlx5_flow_action_release,
 755         .tunnel_item_release = mlx5_flow_item_release,
 756         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 757 };
 758
 759 /* Convert FDIR request to Generic flow. */
 760 struct mlx5_fdir {
 761         struct rte_flow_attr attr;
 762         struct rte_flow_item items[4];
 763         struct rte_flow_item_eth l2;
 764         struct rte_flow_item_eth l2_mask;
 765         union {
 766                 struct rte_flow_item_ipv4 ipv4;
 767                 struct rte_flow_item_ipv6 ipv6;
 768         } l3;
 769         union {
 770                 struct rte_flow_item_ipv4 ipv4;
 771                 struct rte_flow_item_ipv6 ipv6;
 772         } l3_mask;
 773         union {
 774                 struct rte_flow_item_udp udp;
 775                 struct rte_flow_item_tcp tcp;
 776         } l4;
 777         union {
 778                 struct rte_flow_item_udp udp;
 779                 struct rte_flow_item_tcp tcp;
 780         } l4_mask;
 781         struct rte_flow_action actions[2];
 782         struct rte_flow_action_queue queue;
 783 };
 784
 785 /* Tunnel information. */
 786 struct mlx5_flow_tunnel_info {
 787         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 788         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 789 };
 790
 791 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 792         {
 793                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 794                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 795         },
 796         {
 797                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 798                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 799         },
 800         {
 801                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 802                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 803         },
 804         {
 805                 .tunnel = MLX5_FLOW_LAYER_GRE,
 806                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 807         },
 808         {
 809                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 810                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 811         },
 812         {
 813                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 814                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 815         },
 816         {
 817                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 818                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 819         },
 820         {
 821                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 822                 .ptype = RTE_PTYPE_TUNNEL_IP,
 823         },
 824         {
 825                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 826                 .ptype = RTE_PTYPE_TUNNEL_IP,
 827         },
 828         {
 829                 .tunnel = MLX5_FLOW_LAYER_GTP,
 830                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 831         },
 832 };
 833
 834 /**
 835  * Translate tag ID to register.
 836  *
 837  * @param[in] dev
 838  *   Pointer to the Ethernet device structure.
 839  * @param[in] feature
 840  *   The feature that request the register.
 841  * @param[in] id
 842  *   The request register ID.
 843  * @param[out] error
 844  *   Error description in case of any.
 845  *
 846  * @return
 847  *   The request register on success, a negative errno
 848  *   value otherwise and rte_errno is set.
 849  */
 850 int
 851 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 852                      enum mlx5_feature_name feature,
 853                      uint32_t id,
 854                      struct rte_flow_error *error)
 855 {
 856         struct mlx5_priv *priv = dev->data->dev_private;
 857         struct mlx5_dev_config *config = &priv->config;
 858         enum modify_reg start_reg;
 859         bool skip_mtr_reg = false;
 860
 861         switch (feature) {
 862         case MLX5_HAIRPIN_RX:
 863                 return REG_B;
 864         case MLX5_HAIRPIN_TX:
 865                 return REG_A;
 866         case MLX5_METADATA_RX:
 867                 switch (config->dv_xmeta_en) {
 868                 case MLX5_XMETA_MODE_LEGACY:
 869                         return REG_B;
 870                 case MLX5_XMETA_MODE_META16:
 871                         return REG_C_0;
 872                 case MLX5_XMETA_MODE_META32:
 873                         return REG_C_1;
 874                 }
 875                 break;
 876         case MLX5_METADATA_TX:
 877                 return REG_A;
 878         case MLX5_METADATA_FDB:
 879                 switch (config->dv_xmeta_en) {
 880                 case MLX5_XMETA_MODE_LEGACY:
 881                         return REG_NON;
 882                 case MLX5_XMETA_MODE_META16:
 883                         return REG_C_0;
 884                 case MLX5_XMETA_MODE_META32:
 885                         return REG_C_1;
 886                 }
 887                 break;
 888         case MLX5_FLOW_MARK:
 889                 switch (config->dv_xmeta_en) {
 890                 case MLX5_XMETA_MODE_LEGACY:
 891                         return REG_NON;
 892                 case MLX5_XMETA_MODE_META16:
 893                         return REG_C_1;
 894                 case MLX5_XMETA_MODE_META32:
 895                         return REG_C_0;
 896                 }
 897                 break;
 898         case MLX5_MTR_SFX:
 899                 /*
 900                  * If meter color and flow match share one register, flow match
 901                  * should use the meter color register for match.
 902                  */
 903                 if (priv->mtr_reg_share)
 904                         return priv->mtr_color_reg;
 905                 else
 906                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 907                                REG_C_3;
 908         case MLX5_MTR_COLOR:
 909                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 910                 return priv->mtr_color_reg;
 911         case MLX5_COPY_MARK:
 912                 /*
 913                  * Metadata COPY_MARK register using is in meter suffix sub
 914                  * flow while with meter. It's safe to share the same register.
 915                  */
 916                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 917         case MLX5_APP_TAG:
 918                 /*
 919                  * If meter is enable, it will engage the register for color
 920                  * match and flow match. If meter color match is not using the
 921                  * REG_C_2, need to skip the REG_C_x be used by meter color
 922                  * match.
 923                  * If meter is disable, free to use all available registers.
 924                  */
 925                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 926                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 927                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 928                 if (id > (REG_C_7 - start_reg))
 929                         return rte_flow_error_set(error, EINVAL,
 930                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 931                                                   NULL, "invalid tag id");
 932                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 933                         return rte_flow_error_set(error, ENOTSUP,
 934                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 935                                                   NULL, "unsupported tag id");
 936                 /*
 937                  * This case means meter is using the REG_C_x great than 2.
 938                  * Take care not to conflict with meter color REG_C_x.
 939                  * If the available index REG_C_y >= REG_C_x, skip the
 940                  * color register.
 941                  */
 942                 if (skip_mtr_reg && config->flow_mreg_c
 943                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 944                         if (id >= (REG_C_7 - start_reg))
 945                                 return rte_flow_error_set(error, EINVAL,
 946                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 947                                                         NULL, "invalid tag id");
 948                         if (config->flow_mreg_c
 949                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 950                                 return config->flow_mreg_c
 951                                                [id + 1 + start_reg - REG_C_0];
 952                         return rte_flow_error_set(error, ENOTSUP,
 953                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 954                                                   NULL, "unsupported tag id");
 955                 }
 956                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 957         }
 958         MLX5_ASSERT(false);
 959         return rte_flow_error_set(error, EINVAL,
 960                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 961                                   NULL, "invalid feature name");
 962 }
 963
 964 /**
 965  * Check extensive flow metadata register support.
 966  *
 967  * @param dev
 968  *   Pointer to rte_eth_dev structure.
 969  *
 970  * @return
 971  *   True if device supports extensive flow metadata register, otherwise false.
 972  */
 973 bool
 974 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 975 {
 976         struct mlx5_priv *priv = dev->data->dev_private;
 977         struct mlx5_dev_config *config = &priv->config;
 978
 979         /*
 980          * Having available reg_c can be regarded inclusively as supporting
 981          * extensive flow metadata register, which could mean,
 982          * - metadata register copy action by modify header.
 983          * - 16 modify header actions is supported.
 984          * - reg_c's are preserved across different domain (FDB and NIC) on
 985          *   packet loopback by flow lookup miss.
 986          */
 987         return config->flow_mreg_c[2] != REG_NON;
 988 }
 989
 990 /**
 991  * Verify the @p item specifications (spec, last, mask) are compatible with the
 992  * NIC capabilities.
 993  *
 994  * @param[in] item
 995  *   Item specification.
 996  * @param[in] mask
 997  *   @p item->mask or flow default bit-masks.
 998  * @param[in] nic_mask
 999  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1000  * @param[in] size
1001  *   Bit-masks size in bytes.
1002  * @param[in] range_accepted
1003  *   True if range of values is accepted for specific fields, false otherwise.
1004  * @param[out] error
1005  *   Pointer to error structure.
1006  *
1007  * @return
1008  *   0 on success, a negative errno value otherwise and rte_errno is set.
1009  */
1010 int
1011 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1012                           const uint8_t *mask,
1013                           const uint8_t *nic_mask,
1014                           unsigned int size,
1015                           bool range_accepted,
1016                           struct rte_flow_error *error)
1017 {
1018         unsigned int i;
1019
1020         MLX5_ASSERT(nic_mask);
1021         for (i = 0; i < size; ++i)
1022                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1023                         return rte_flow_error_set(error, ENOTSUP,
1024                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1025                                                   item,
1026                                                   "mask enables non supported"
1027                                                   " bits");
1028         if (!item->spec && (item->mask || item->last))
1029                 return rte_flow_error_set(error, EINVAL,
1030                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1031                                           "mask/last without a spec is not"
1032                                           " supported");
1033         if (item->spec && item->last && !range_accepted) {
1034                 uint8_t spec[size];
1035                 uint8_t last[size];
1036                 unsigned int i;
1037                 int ret;
1038
1039                 for (i = 0; i < size; ++i) {
1040                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1041                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1042                 }
1043                 ret = memcmp(spec, last, size);
1044                 if (ret != 0)
1045                         return rte_flow_error_set(error, EINVAL,
1046                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1047                                                   item,
1048                                                   "range is not valid");
1049         }
1050         return 0;
1051 }
1052
1053 /**
1054  * Adjust the hash fields according to the @p flow information.
1055  *
1056  * @param[in] dev_flow.
1057  *   Pointer to the mlx5_flow.
1058  * @param[in] tunnel
1059  *   1 when the hash field is for a tunnel item.
1060  * @param[in] layer_types
1061  *   ETH_RSS_* types.
1062  * @param[in] hash_fields
1063  *   Item hash fields.
1064  *
1065  * @return
1066  *   The hash fields that should be used.
1067  */
1068 uint64_t
1069 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1070                             int tunnel __rte_unused, uint64_t layer_types,
1071                             uint64_t hash_fields)
1072 {
1073 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1074         int rss_request_inner = rss_desc->level >= 2;
1075
1076         /* Check RSS hash level for tunnel. */
1077         if (tunnel && rss_request_inner)
1078                 hash_fields |= IBV_RX_HASH_INNER;
1079         else if (tunnel || rss_request_inner)
1080                 return 0;
1081 #endif
1082         /* Check if requested layer matches RSS hash fields. */
1083         if (!(rss_desc->types & layer_types))
1084                 return 0;
1085         return hash_fields;
1086 }
1087
1088 /**
1089  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1090  * if several tunnel rules are used on this queue, the tunnel ptype will be
1091  * cleared.
1092  *
1093  * @param rxq_ctrl
1094  *   Rx queue to update.
1095  */
1096 static void
1097 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1098 {
1099         unsigned int i;
1100         uint32_t tunnel_ptype = 0;
1101
1102         /* Look up for the ptype to use. */
1103         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1104                 if (!rxq_ctrl->flow_tunnels_n[i])
1105                         continue;
1106                 if (!tunnel_ptype) {
1107                         tunnel_ptype = tunnels_info[i].ptype;
1108                 } else {
1109                         tunnel_ptype = 0;
1110                         break;
1111                 }
1112         }
1113         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1114 }
1115
1116 /**
1117  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1118  * flow.
1119  *
1120  * @param[in] dev
1121  *   Pointer to the Ethernet device structure.
1122  * @param[in] dev_handle
1123  *   Pointer to device flow handle structure.
1124  */
1125 static void
1126 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1127                        struct mlx5_flow_handle *dev_handle)
1128 {
1129         struct mlx5_priv *priv = dev->data->dev_private;
1130         const int mark = dev_handle->mark;
1131         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1132         struct mlx5_hrxq *hrxq;
1133         unsigned int i;
1134
1135         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1136                 return;
1137         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1138                               dev_handle->rix_hrxq);
1139         if (!hrxq)
1140                 return;
1141         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1142                 int idx = hrxq->ind_table->queues[i];
1143                 struct mlx5_rxq_ctrl *rxq_ctrl =
1144                         container_of((*priv->rxqs)[idx],
1145                                      struct mlx5_rxq_ctrl, rxq);
1146
1147                 /*
1148                  * To support metadata register copy on Tx loopback,
1149                  * this must be always enabled (metadata may arive
1150                  * from other port - not from local flows only.
1151                  */
1152                 if (priv->config.dv_flow_en &&
1153                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1154                     mlx5_flow_ext_mreg_supported(dev)) {
1155                         rxq_ctrl->rxq.mark = 1;
1156                         rxq_ctrl->flow_mark_n = 1;
1157                 } else if (mark) {
1158                         rxq_ctrl->rxq.mark = 1;
1159                         rxq_ctrl->flow_mark_n++;
1160                 }
1161                 if (tunnel) {
1162                         unsigned int j;
1163
1164                         /* Increase the counter matching the flow. */
1165                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1166                                 if ((tunnels_info[j].tunnel &
1167                                      dev_handle->layers) ==
1168                                     tunnels_info[j].tunnel) {
1169                                         rxq_ctrl->flow_tunnels_n[j]++;
1170                                         break;
1171                                 }
1172                         }
1173                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1174                 }
1175         }
1176 }
1177
1178 /**
1179  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1180  *
1181  * @param[in] dev
1182  *   Pointer to the Ethernet device structure.
1183  * @param[in] flow
1184  *   Pointer to flow structure.
1185  */
1186 static void
1187 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1188 {
1189         struct mlx5_priv *priv = dev->data->dev_private;
1190         uint32_t handle_idx;
1191         struct mlx5_flow_handle *dev_handle;
1192
1193         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1194                        handle_idx, dev_handle, next)
1195                 flow_drv_rxq_flags_set(dev, dev_handle);
1196 }
1197
1198 /**
1199  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1200  * device flow if no other flow uses it with the same kind of request.
1201  *
1202  * @param dev
1203  *   Pointer to Ethernet device.
1204  * @param[in] dev_handle
1205  *   Pointer to the device flow handle structure.
1206  */
1207 static void
1208 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1209                         struct mlx5_flow_handle *dev_handle)
1210 {
1211         struct mlx5_priv *priv = dev->data->dev_private;
1212         const int mark = dev_handle->mark;
1213         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1214         struct mlx5_hrxq *hrxq;
1215         unsigned int i;
1216
1217         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1218                 return;
1219         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1220                               dev_handle->rix_hrxq);
1221         if (!hrxq)
1222                 return;
1223         MLX5_ASSERT(dev->data->dev_started);
1224         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1225                 int idx = hrxq->ind_table->queues[i];
1226                 struct mlx5_rxq_ctrl *rxq_ctrl =
1227                         container_of((*priv->rxqs)[idx],
1228                                      struct mlx5_rxq_ctrl, rxq);
1229
1230                 if (priv->config.dv_flow_en &&
1231                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1232                     mlx5_flow_ext_mreg_supported(dev)) {
1233                         rxq_ctrl->rxq.mark = 1;
1234                         rxq_ctrl->flow_mark_n = 1;
1235                 } else if (mark) {
1236                         rxq_ctrl->flow_mark_n--;
1237                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1238                 }
1239                 if (tunnel) {
1240                         unsigned int j;
1241
1242                         /* Decrease the counter matching the flow. */
1243                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1244                                 if ((tunnels_info[j].tunnel &
1245                                      dev_handle->layers) ==
1246                                     tunnels_info[j].tunnel) {
1247                                         rxq_ctrl->flow_tunnels_n[j]--;
1248                                         break;
1249                                 }
1250                         }
1251                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1252                 }
1253         }
1254 }
1255
1256 /**
1257  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1258  * @p flow if no other flow uses it with the same kind of request.
1259  *
1260  * @param dev
1261  *   Pointer to Ethernet device.
1262  * @param[in] flow
1263  *   Pointer to the flow.
1264  */
1265 static void
1266 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1267 {
1268         struct mlx5_priv *priv = dev->data->dev_private;
1269         uint32_t handle_idx;
1270         struct mlx5_flow_handle *dev_handle;
1271
1272         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1273                        handle_idx, dev_handle, next)
1274                 flow_drv_rxq_flags_trim(dev, dev_handle);
1275 }
1276
1277 /**
1278  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1279  *
1280  * @param dev
1281  *   Pointer to Ethernet device.
1282  */
1283 static void
1284 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1285 {
1286         struct mlx5_priv *priv = dev->data->dev_private;
1287         unsigned int i;
1288
1289         for (i = 0; i != priv->rxqs_n; ++i) {
1290                 struct mlx5_rxq_ctrl *rxq_ctrl;
1291                 unsigned int j;
1292
1293                 if (!(*priv->rxqs)[i])
1294                         continue;
1295                 rxq_ctrl = container_of((*priv->rxqs)[i],
1296                                         struct mlx5_rxq_ctrl, rxq);
1297                 rxq_ctrl->flow_mark_n = 0;
1298                 rxq_ctrl->rxq.mark = 0;
1299                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1300                         rxq_ctrl->flow_tunnels_n[j] = 0;
1301                 rxq_ctrl->rxq.tunnel = 0;
1302         }
1303 }
1304
1305 /**
1306  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1307  *
1308  * @param[in] dev
1309  *   Pointer to the Ethernet device structure.
1310  */
1311 void
1312 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1313 {
1314         struct mlx5_priv *priv = dev->data->dev_private;
1315         struct mlx5_rxq_data *data;
1316         unsigned int i;
1317
1318         for (i = 0; i != priv->rxqs_n; ++i) {
1319                 if (!(*priv->rxqs)[i])
1320                         continue;
1321                 data = (*priv->rxqs)[i];
1322                 if (!rte_flow_dynf_metadata_avail()) {
1323                         data->dynf_meta = 0;
1324                         data->flow_meta_mask = 0;
1325                         data->flow_meta_offset = -1;
1326                 } else {
1327                         data->dynf_meta = 1;
1328                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1329                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1330                 }
1331         }
1332 }
1333
1334 /*
1335  * return a pointer to the desired action in the list of actions.
1336  *
1337  * @param[in] actions
1338  *   The list of actions to search the action in.
1339  * @param[in] action
1340  *   The action to find.
1341  *
1342  * @return
1343  *   Pointer to the action in the list, if found. NULL otherwise.
1344  */
1345 const struct rte_flow_action *
1346 mlx5_flow_find_action(const struct rte_flow_action *actions,
1347                       enum rte_flow_action_type action)
1348 {
1349         if (actions == NULL)
1350                 return NULL;
1351         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1352                 if (actions->type == action)
1353                         return actions;
1354         return NULL;
1355 }
1356
1357 /*
1358  * Validate the flag action.
1359  *
1360  * @param[in] action_flags
1361  *   Bit-fields that holds the actions detected until now.
1362  * @param[in] attr
1363  *   Attributes of flow that includes this action.
1364  * @param[out] error
1365  *   Pointer to error structure.
1366  *
1367  * @return
1368  *   0 on success, a negative errno value otherwise and rte_errno is set.
1369  */
1370 int
1371 mlx5_flow_validate_action_flag(uint64_t action_flags,
1372                                const struct rte_flow_attr *attr,
1373                                struct rte_flow_error *error)
1374 {
1375         if (action_flags & MLX5_FLOW_ACTION_MARK)
1376                 return rte_flow_error_set(error, EINVAL,
1377                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1378                                           "can't mark and flag in same flow");
1379         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1380                 return rte_flow_error_set(error, EINVAL,
1381                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1382                                           "can't have 2 flag"
1383                                           " actions in same flow");
1384         if (attr->egress)
1385                 return rte_flow_error_set(error, ENOTSUP,
1386                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1387                                           "flag action not supported for "
1388                                           "egress");
1389         return 0;
1390 }
1391
1392 /*
1393  * Validate the mark action.
1394  *
1395  * @param[in] action
1396  *   Pointer to the queue action.
1397  * @param[in] action_flags
1398  *   Bit-fields that holds the actions detected until now.
1399  * @param[in] attr
1400  *   Attributes of flow that includes this action.
1401  * @param[out] error
1402  *   Pointer to error structure.
1403  *
1404  * @return
1405  *   0 on success, a negative errno value otherwise and rte_errno is set.
1406  */
1407 int
1408 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1409                                uint64_t action_flags,
1410                                const struct rte_flow_attr *attr,
1411                                struct rte_flow_error *error)
1412 {
1413         const struct rte_flow_action_mark *mark = action->conf;
1414
1415         if (!mark)
1416                 return rte_flow_error_set(error, EINVAL,
1417                                           RTE_FLOW_ERROR_TYPE_ACTION,
1418                                           action,
1419                                           "configuration cannot be null");
1420         if (mark->id >= MLX5_FLOW_MARK_MAX)
1421                 return rte_flow_error_set(error, EINVAL,
1422                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1423                                           &mark->id,
1424                                           "mark id must in 0 <= id < "
1425                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1426         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1427                 return rte_flow_error_set(error, EINVAL,
1428                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1429                                           "can't flag and mark in same flow");
1430         if (action_flags & MLX5_FLOW_ACTION_MARK)
1431                 return rte_flow_error_set(error, EINVAL,
1432                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1433                                           "can't have 2 mark actions in same"
1434                                           " flow");
1435         if (attr->egress)
1436                 return rte_flow_error_set(error, ENOTSUP,
1437                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1438                                           "mark action not supported for "
1439                                           "egress");
1440         return 0;
1441 }
1442
1443 /*
1444  * Validate the drop action.
1445  *
1446  * @param[in] action_flags
1447  *   Bit-fields that holds the actions detected until now.
1448  * @param[in] attr
1449  *   Attributes of flow that includes this action.
1450  * @param[out] error
1451  *   Pointer to error structure.
1452  *
1453  * @return
1454  *   0 on success, a negative errno value otherwise and rte_errno is set.
1455  */
1456 int
1457 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1458                                const struct rte_flow_attr *attr,
1459                                struct rte_flow_error *error)
1460 {
1461         if (attr->egress)
1462                 return rte_flow_error_set(error, ENOTSUP,
1463                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1464                                           "drop action not supported for "
1465                                           "egress");
1466         return 0;
1467 }
1468
1469 /*
1470  * Validate the queue action.
1471  *
1472  * @param[in] action
1473  *   Pointer to the queue action.
1474  * @param[in] action_flags
1475  *   Bit-fields that holds the actions detected until now.
1476  * @param[in] dev
1477  *   Pointer to the Ethernet device structure.
1478  * @param[in] attr
1479  *   Attributes of flow that includes this action.
1480  * @param[out] error
1481  *   Pointer to error structure.
1482  *
1483  * @return
1484  *   0 on success, a negative errno value otherwise and rte_errno is set.
1485  */
1486 int
1487 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1488                                 uint64_t action_flags,
1489                                 struct rte_eth_dev *dev,
1490                                 const struct rte_flow_attr *attr,
1491                                 struct rte_flow_error *error)
1492 {
1493         struct mlx5_priv *priv = dev->data->dev_private;
1494         const struct rte_flow_action_queue *queue = action->conf;
1495
1496         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1497                 return rte_flow_error_set(error, EINVAL,
1498                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1499                                           "can't have 2 fate actions in"
1500                                           " same flow");
1501         if (!priv->rxqs_n)
1502                 return rte_flow_error_set(error, EINVAL,
1503                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1504                                           NULL, "No Rx queues configured");
1505         if (queue->index >= priv->rxqs_n)
1506                 return rte_flow_error_set(error, EINVAL,
1507                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1508                                           &queue->index,
1509                                           "queue index out of range");
1510         if (!(*priv->rxqs)[queue->index])
1511                 return rte_flow_error_set(error, EINVAL,
1512                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1513                                           &queue->index,
1514                                           "queue is not configured");
1515         if (attr->egress)
1516                 return rte_flow_error_set(error, ENOTSUP,
1517                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1518                                           "queue action not supported for "
1519                                           "egress");
1520         return 0;
1521 }
1522
1523 /*
1524  * Validate the rss action.
1525  *
1526  * @param[in] dev
1527  *   Pointer to the Ethernet device structure.
1528  * @param[in] action
1529  *   Pointer to the queue action.
1530  * @param[out] error
1531  *   Pointer to error structure.
1532  *
1533  * @return
1534  *   0 on success, a negative errno value otherwise and rte_errno is set.
1535  */
1536 int
1537 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1538                          const struct rte_flow_action *action,
1539                          struct rte_flow_error *error)
1540 {
1541         struct mlx5_priv *priv = dev->data->dev_private;
1542         const struct rte_flow_action_rss *rss = action->conf;
1543         unsigned int i;
1544
1545         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1546             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1547                 return rte_flow_error_set(error, ENOTSUP,
1548                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1549                                           &rss->func,
1550                                           "RSS hash function not supported");
1551 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1552         if (rss->level > 2)
1553 #else
1554         if (rss->level > 1)
1555 #endif
1556                 return rte_flow_error_set(error, ENOTSUP,
1557                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1558                                           &rss->level,
1559                                           "tunnel RSS is not supported");
1560         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1561         if (rss->key_len == 0 && rss->key != NULL)
1562                 return rte_flow_error_set(error, ENOTSUP,
1563                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1564                                           &rss->key_len,
1565                                           "RSS hash key length 0");
1566         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1567                 return rte_flow_error_set(error, ENOTSUP,
1568                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1569                                           &rss->key_len,
1570                                           "RSS hash key too small");
1571         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1572                 return rte_flow_error_set(error, ENOTSUP,
1573                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1574                                           &rss->key_len,
1575                                           "RSS hash key too large");
1576         if (rss->queue_num > priv->config.ind_table_max_size)
1577                 return rte_flow_error_set(error, ENOTSUP,
1578                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1579                                           &rss->queue_num,
1580                                           "number of queues too large");
1581         if (rss->types & MLX5_RSS_HF_MASK)
1582                 return rte_flow_error_set(error, ENOTSUP,
1583                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1584                                           &rss->types,
1585                                           "some RSS protocols are not"
1586                                           " supported");
1587         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1588             !(rss->types & ETH_RSS_IP))
1589                 return rte_flow_error_set(error, EINVAL,
1590                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1591                                           "L3 partial RSS requested but L3 RSS"
1592                                           " type not specified");
1593         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1594             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1595                 return rte_flow_error_set(error, EINVAL,
1596                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1597                                           "L4 partial RSS requested but L4 RSS"
1598                                           " type not specified");
1599         if (!priv->rxqs_n)
1600                 return rte_flow_error_set(error, EINVAL,
1601                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1602                                           NULL, "No Rx queues configured");
1603         if (!rss->queue_num)
1604                 return rte_flow_error_set(error, EINVAL,
1605                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1606                                           NULL, "No queues configured");
1607         for (i = 0; i != rss->queue_num; ++i) {
1608                 if (rss->queue[i] >= priv->rxqs_n)
1609                         return rte_flow_error_set
1610                                 (error, EINVAL,
1611                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1612                                  &rss->queue[i], "queue index out of range");
1613                 if (!(*priv->rxqs)[rss->queue[i]])
1614                         return rte_flow_error_set
1615                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1616                                  &rss->queue[i], "queue is not configured");
1617         }
1618         return 0;
1619 }
1620
1621 /*
1622  * Validate the rss action.
1623  *
1624  * @param[in] action
1625  *   Pointer to the queue action.
1626  * @param[in] action_flags
1627  *   Bit-fields that holds the actions detected until now.
1628  * @param[in] dev
1629  *   Pointer to the Ethernet device structure.
1630  * @param[in] attr
1631  *   Attributes of flow that includes this action.
1632  * @param[in] item_flags
1633  *   Items that were detected.
1634  * @param[out] error
1635  *   Pointer to error structure.
1636  *
1637  * @return
1638  *   0 on success, a negative errno value otherwise and rte_errno is set.
1639  */
1640 int
1641 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1642                               uint64_t action_flags,
1643                               struct rte_eth_dev *dev,
1644                               const struct rte_flow_attr *attr,
1645                               uint64_t item_flags,
1646                               struct rte_flow_error *error)
1647 {
1648         const struct rte_flow_action_rss *rss = action->conf;
1649         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1650         int ret;
1651
1652         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1653                 return rte_flow_error_set(error, EINVAL,
1654                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1655                                           "can't have 2 fate actions"
1656                                           " in same flow");
1657         ret = mlx5_validate_action_rss(dev, action, error);
1658         if (ret)
1659                 return ret;
1660         if (attr->egress)
1661                 return rte_flow_error_set(error, ENOTSUP,
1662                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1663                                           "rss action not supported for "
1664                                           "egress");
1665         if (rss->level > 1 && !tunnel)
1666                 return rte_flow_error_set(error, EINVAL,
1667                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1668                                           "inner RSS is not supported for "
1669                                           "non-tunnel flows");
1670         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1671             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1672                 return rte_flow_error_set(error, EINVAL,
1673                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1674                                           "RSS on eCPRI is not supported now");
1675         }
1676         return 0;
1677 }
1678
1679 /*
1680  * Validate the default miss action.
1681  *
1682  * @param[in] action_flags
1683  *   Bit-fields that holds the actions detected until now.
1684  * @param[out] error
1685  *   Pointer to error structure.
1686  *
1687  * @return
1688  *   0 on success, a negative errno value otherwise and rte_errno is set.
1689  */
1690 int
1691 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1692                                 const struct rte_flow_attr *attr,
1693                                 struct rte_flow_error *error)
1694 {
1695         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1696                 return rte_flow_error_set(error, EINVAL,
1697                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1698                                           "can't have 2 fate actions in"
1699                                           " same flow");
1700         if (attr->egress)
1701                 return rte_flow_error_set(error, ENOTSUP,
1702                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1703                                           "default miss action not supported "
1704                                           "for egress");
1705         if (attr->group)
1706                 return rte_flow_error_set(error, ENOTSUP,
1707                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1708                                           "only group 0 is supported");
1709         if (attr->transfer)
1710                 return rte_flow_error_set(error, ENOTSUP,
1711                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1712                                           NULL, "transfer is not supported");
1713         return 0;
1714 }
1715
1716 /*
1717  * Validate the count action.
1718  *
1719  * @param[in] dev
1720  *   Pointer to the Ethernet device structure.
1721  * @param[in] attr
1722  *   Attributes of flow that includes this action.
1723  * @param[out] error
1724  *   Pointer to error structure.
1725  *
1726  * @return
1727  *   0 on success, a negative errno value otherwise and rte_errno is set.
1728  */
1729 int
1730 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1731                                 const struct rte_flow_attr *attr,
1732                                 struct rte_flow_error *error)
1733 {
1734         if (attr->egress)
1735                 return rte_flow_error_set(error, ENOTSUP,
1736                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1737                                           "count action not supported for "
1738                                           "egress");
1739         return 0;
1740 }
1741
1742 /**
1743  * Verify the @p attributes will be correctly understood by the NIC and store
1744  * them in the @p flow if everything is correct.
1745  *
1746  * @param[in] dev
1747  *   Pointer to the Ethernet device structure.
1748  * @param[in] attributes
1749  *   Pointer to flow attributes
1750  * @param[out] error
1751  *   Pointer to error structure.
1752  *
1753  * @return
1754  *   0 on success, a negative errno value otherwise and rte_errno is set.
1755  */
1756 int
1757 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1758                               const struct rte_flow_attr *attributes,
1759                               struct rte_flow_error *error)
1760 {
1761         struct mlx5_priv *priv = dev->data->dev_private;
1762         uint32_t priority_max = priv->config.flow_prio - 1;
1763
1764         if (attributes->group)
1765                 return rte_flow_error_set(error, ENOTSUP,
1766                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1767                                           NULL, "groups is not supported");
1768         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1769             attributes->priority >= priority_max)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1772                                           NULL, "priority out of range");
1773         if (attributes->egress)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1776                                           "egress is not supported");
1777         if (attributes->transfer && !priv->config.dv_esw_en)
1778                 return rte_flow_error_set(error, ENOTSUP,
1779                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1780                                           NULL, "transfer is not supported");
1781         if (!attributes->ingress)
1782                 return rte_flow_error_set(error, EINVAL,
1783                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1784                                           NULL,
1785                                           "ingress attribute is mandatory");
1786         return 0;
1787 }
1788
1789 /**
1790  * Validate ICMP6 item.
1791  *
1792  * @param[in] item
1793  *   Item specification.
1794  * @param[in] item_flags
1795  *   Bit-fields that holds the items detected until now.
1796  * @param[in] ext_vlan_sup
1797  *   Whether extended VLAN features are supported or not.
1798  * @param[out] error
1799  *   Pointer to error structure.
1800  *
1801  * @return
1802  *   0 on success, a negative errno value otherwise and rte_errno is set.
1803  */
1804 int
1805 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1806                                uint64_t item_flags,
1807                                uint8_t target_protocol,
1808                                struct rte_flow_error *error)
1809 {
1810         const struct rte_flow_item_icmp6 *mask = item->mask;
1811         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1812         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1813                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1814         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1815                                       MLX5_FLOW_LAYER_OUTER_L4;
1816         int ret;
1817
1818         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1819                 return rte_flow_error_set(error, EINVAL,
1820                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1821                                           "protocol filtering not compatible"
1822                                           " with ICMP6 layer");
1823         if (!(item_flags & l3m))
1824                 return rte_flow_error_set(error, EINVAL,
1825                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1826                                           "IPv6 is mandatory to filter on"
1827                                           " ICMP6");
1828         if (item_flags & l4m)
1829                 return rte_flow_error_set(error, EINVAL,
1830                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1831                                           "multiple L4 layers not supported");
1832         if (!mask)
1833                 mask = &rte_flow_item_icmp6_mask;
1834         ret = mlx5_flow_item_acceptable
1835                 (item, (const uint8_t *)mask,
1836                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1837                  sizeof(struct rte_flow_item_icmp6),
1838                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1839         if (ret < 0)
1840                 return ret;
1841         return 0;
1842 }
1843
1844 /**
1845  * Validate ICMP item.
1846  *
1847  * @param[in] item
1848  *   Item specification.
1849  * @param[in] item_flags
1850  *   Bit-fields that holds the items detected until now.
1851  * @param[out] error
1852  *   Pointer to error structure.
1853  *
1854  * @return
1855  *   0 on success, a negative errno value otherwise and rte_errno is set.
1856  */
1857 int
1858 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1859                              uint64_t item_flags,
1860                              uint8_t target_protocol,
1861                              struct rte_flow_error *error)
1862 {
1863         const struct rte_flow_item_icmp *mask = item->mask;
1864         const struct rte_flow_item_icmp nic_mask = {
1865                 .hdr.icmp_type = 0xff,
1866                 .hdr.icmp_code = 0xff,
1867                 .hdr.icmp_ident = RTE_BE16(0xffff),
1868                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1869         };
1870         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1871         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1872                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1873         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1874                                       MLX5_FLOW_LAYER_OUTER_L4;
1875         int ret;
1876
1877         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1878                 return rte_flow_error_set(error, EINVAL,
1879                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1880                                           "protocol filtering not compatible"
1881                                           " with ICMP layer");
1882         if (!(item_flags & l3m))
1883                 return rte_flow_error_set(error, EINVAL,
1884                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1885                                           "IPv4 is mandatory to filter"
1886                                           " on ICMP");
1887         if (item_flags & l4m)
1888                 return rte_flow_error_set(error, EINVAL,
1889                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1890                                           "multiple L4 layers not supported");
1891         if (!mask)
1892                 mask = &nic_mask;
1893         ret = mlx5_flow_item_acceptable
1894                 (item, (const uint8_t *)mask,
1895                  (const uint8_t *)&nic_mask,
1896                  sizeof(struct rte_flow_item_icmp),
1897                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1898         if (ret < 0)
1899                 return ret;
1900         return 0;
1901 }
1902
1903 /**
1904  * Validate Ethernet item.
1905  *
1906  * @param[in] item
1907  *   Item specification.
1908  * @param[in] item_flags
1909  *   Bit-fields that holds the items detected until now.
1910  * @param[out] error
1911  *   Pointer to error structure.
1912  *
1913  * @return
1914  *   0 on success, a negative errno value otherwise and rte_errno is set.
1915  */
1916 int
1917 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1918                             uint64_t item_flags, bool ext_vlan_sup,
1919                             struct rte_flow_error *error)
1920 {
1921         const struct rte_flow_item_eth *mask = item->mask;
1922         const struct rte_flow_item_eth nic_mask = {
1923                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1924                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1925                 .type = RTE_BE16(0xffff),
1926                 .has_vlan = ext_vlan_sup ? 1 : 0,
1927         };
1928         int ret;
1929         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1930         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1931                                        MLX5_FLOW_LAYER_OUTER_L2;
1932
1933         if (item_flags & ethm)
1934                 return rte_flow_error_set(error, ENOTSUP,
1935                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1936                                           "multiple L2 layers not supported");
1937         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1938             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1939                 return rte_flow_error_set(error, EINVAL,
1940                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1941                                           "L2 layer should not follow "
1942                                           "L3 layers");
1943         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1944             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1945                 return rte_flow_error_set(error, EINVAL,
1946                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1947                                           "L2 layer should not follow VLAN");
1948         if (!mask)
1949                 mask = &rte_flow_item_eth_mask;
1950         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1951                                         (const uint8_t *)&nic_mask,
1952                                         sizeof(struct rte_flow_item_eth),
1953                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1954         return ret;
1955 }
1956
1957 /**
1958  * Validate VLAN item.
1959  *
1960  * @param[in] item
1961  *   Item specification.
1962  * @param[in] item_flags
1963  *   Bit-fields that holds the items detected until now.
1964  * @param[in] dev
1965  *   Ethernet device flow is being created on.
1966  * @param[out] error
1967  *   Pointer to error structure.
1968  *
1969  * @return
1970  *   0 on success, a negative errno value otherwise and rte_errno is set.
1971  */
1972 int
1973 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1974                              uint64_t item_flags,
1975                              struct rte_eth_dev *dev,
1976                              struct rte_flow_error *error)
1977 {
1978         const struct rte_flow_item_vlan *spec = item->spec;
1979         const struct rte_flow_item_vlan *mask = item->mask;
1980         const struct rte_flow_item_vlan nic_mask = {
1981                 .tci = RTE_BE16(UINT16_MAX),
1982                 .inner_type = RTE_BE16(UINT16_MAX),
1983         };
1984         uint16_t vlan_tag = 0;
1985         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1986         int ret;
1987         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1988                                         MLX5_FLOW_LAYER_INNER_L4) :
1989                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1990                                         MLX5_FLOW_LAYER_OUTER_L4);
1991         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1992                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1993
1994         if (item_flags & vlanm)
1995                 return rte_flow_error_set(error, EINVAL,
1996                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1997                                           "multiple VLAN layers not supported");
1998         else if ((item_flags & l34m) != 0)
1999                 return rte_flow_error_set(error, EINVAL,
2000                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2001                                           "VLAN cannot follow L3/L4 layer");
2002         if (!mask)
2003                 mask = &rte_flow_item_vlan_mask;
2004         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2005                                         (const uint8_t *)&nic_mask,
2006                                         sizeof(struct rte_flow_item_vlan),
2007                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2008         if (ret)
2009                 return ret;
2010         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2011                 struct mlx5_priv *priv = dev->data->dev_private;
2012
2013                 if (priv->vmwa_context) {
2014                         /*
2015                          * Non-NULL context means we have a virtual machine
2016                          * and SR-IOV enabled, we have to create VLAN interface
2017                          * to make hypervisor to setup E-Switch vport
2018                          * context correctly. We avoid creating the multiple
2019                          * VLAN interfaces, so we cannot support VLAN tag mask.
2020                          */
2021                         return rte_flow_error_set(error, EINVAL,
2022                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2023                                                   item,
2024                                                   "VLAN tag mask is not"
2025                                                   " supported in virtual"
2026                                                   " environment");
2027                 }
2028         }
2029         if (spec) {
2030                 vlan_tag = spec->tci;
2031                 vlan_tag &= mask->tci;
2032         }
2033         /*
2034          * From verbs perspective an empty VLAN is equivalent
2035          * to a packet without VLAN layer.
2036          */
2037         if (!vlan_tag)
2038                 return rte_flow_error_set(error, EINVAL,
2039                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2040                                           item->spec,
2041                                           "VLAN cannot be empty");
2042         return 0;
2043 }
2044
2045 /**
2046  * Validate IPV4 item.
2047  *
2048  * @param[in] item
2049  *   Item specification.
2050  * @param[in] item_flags
2051  *   Bit-fields that holds the items detected until now.
2052  * @param[in] last_item
2053  *   Previous validated item in the pattern items.
2054  * @param[in] ether_type
2055  *   Type in the ethernet layer header (including dot1q).
2056  * @param[in] acc_mask
2057  *   Acceptable mask, if NULL default internal default mask
2058  *   will be used to check whether item fields are supported.
2059  * @param[in] range_accepted
2060  *   True if range of values is accepted for specific fields, false otherwise.
2061  * @param[out] error
2062  *   Pointer to error structure.
2063  *
2064  * @return
2065  *   0 on success, a negative errno value otherwise and rte_errno is set.
2066  */
2067 int
2068 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2069                              uint64_t item_flags,
2070                              uint64_t last_item,
2071                              uint16_t ether_type,
2072                              const struct rte_flow_item_ipv4 *acc_mask,
2073                              bool range_accepted,
2074                              struct rte_flow_error *error)
2075 {
2076         const struct rte_flow_item_ipv4 *mask = item->mask;
2077         const struct rte_flow_item_ipv4 *spec = item->spec;
2078         const struct rte_flow_item_ipv4 nic_mask = {
2079                 .hdr = {
2080                         .src_addr = RTE_BE32(0xffffffff),
2081                         .dst_addr = RTE_BE32(0xffffffff),
2082                         .type_of_service = 0xff,
2083                         .next_proto_id = 0xff,
2084                 },
2085         };
2086         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2087         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2088                                       MLX5_FLOW_LAYER_OUTER_L3;
2089         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2090                                       MLX5_FLOW_LAYER_OUTER_L4;
2091         int ret;
2092         uint8_t next_proto = 0xFF;
2093         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2094                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2095                                   MLX5_FLOW_LAYER_INNER_VLAN);
2096
2097         if ((last_item & l2_vlan) && ether_type &&
2098             ether_type != RTE_ETHER_TYPE_IPV4)
2099                 return rte_flow_error_set(error, EINVAL,
2100                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2101                                           "IPv4 cannot follow L2/VLAN layer "
2102                                           "which ether type is not IPv4");
2103         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2104                 if (mask && spec)
2105                         next_proto = mask->hdr.next_proto_id &
2106                                      spec->hdr.next_proto_id;
2107                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2108                         return rte_flow_error_set(error, EINVAL,
2109                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2110                                                   item,
2111                                                   "multiple tunnel "
2112                                                   "not supported");
2113         }
2114         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2115                 return rte_flow_error_set(error, EINVAL,
2116                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2117                                           "wrong tunnel type - IPv6 specified "
2118                                           "but IPv4 item provided");
2119         if (item_flags & l3m)
2120                 return rte_flow_error_set(error, ENOTSUP,
2121                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2122                                           "multiple L3 layers not supported");
2123         else if (item_flags & l4m)
2124                 return rte_flow_error_set(error, EINVAL,
2125                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2126                                           "L3 cannot follow an L4 layer.");
2127         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2128                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2129                 return rte_flow_error_set(error, EINVAL,
2130                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2131                                           "L3 cannot follow an NVGRE layer.");
2132         if (!mask)
2133                 mask = &rte_flow_item_ipv4_mask;
2134         else if (mask->hdr.next_proto_id != 0 &&
2135                  mask->hdr.next_proto_id != 0xff)
2136                 return rte_flow_error_set(error, EINVAL,
2137                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2138                                           "partial mask is not supported"
2139                                           " for protocol");
2140         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2141                                         acc_mask ? (const uint8_t *)acc_mask
2142                                                  : (const uint8_t *)&nic_mask,
2143                                         sizeof(struct rte_flow_item_ipv4),
2144                                         range_accepted, error);
2145         if (ret < 0)
2146                 return ret;
2147         return 0;
2148 }
2149
2150 /**
2151  * Validate IPV6 item.
2152  *
2153  * @param[in] item
2154  *   Item specification.
2155  * @param[in] item_flags
2156  *   Bit-fields that holds the items detected until now.
2157  * @param[in] last_item
2158  *   Previous validated item in the pattern items.
2159  * @param[in] ether_type
2160  *   Type in the ethernet layer header (including dot1q).
2161  * @param[in] acc_mask
2162  *   Acceptable mask, if NULL default internal default mask
2163  *   will be used to check whether item fields are supported.
2164  * @param[out] error
2165  *   Pointer to error structure.
2166  *
2167  * @return
2168  *   0 on success, a negative errno value otherwise and rte_errno is set.
2169  */
2170 int
2171 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2172                              uint64_t item_flags,
2173                              uint64_t last_item,
2174                              uint16_t ether_type,
2175                              const struct rte_flow_item_ipv6 *acc_mask,
2176                              struct rte_flow_error *error)
2177 {
2178         const struct rte_flow_item_ipv6 *mask = item->mask;
2179         const struct rte_flow_item_ipv6 *spec = item->spec;
2180         const struct rte_flow_item_ipv6 nic_mask = {
2181                 .hdr = {
2182                         .src_addr =
2183                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2184                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2185                         .dst_addr =
2186                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2187                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2188                         .vtc_flow = RTE_BE32(0xffffffff),
2189                         .proto = 0xff,
2190                 },
2191         };
2192         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2193         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2194                                       MLX5_FLOW_LAYER_OUTER_L3;
2195         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2196                                       MLX5_FLOW_LAYER_OUTER_L4;
2197         int ret;
2198         uint8_t next_proto = 0xFF;
2199         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2200                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2201                                   MLX5_FLOW_LAYER_INNER_VLAN);
2202
2203         if ((last_item & l2_vlan) && ether_type &&
2204             ether_type != RTE_ETHER_TYPE_IPV6)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2207                                           "IPv6 cannot follow L2/VLAN layer "
2208                                           "which ether type is not IPv6");
2209         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2210                 next_proto = spec->hdr.proto;
2211         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
2212                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2213                         return rte_flow_error_set(error, EINVAL,
2214                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2215                                                   item,
2216                                                   "multiple tunnel "
2217                                                   "not supported");
2218         }
2219         if (next_proto == IPPROTO_HOPOPTS  ||
2220             next_proto == IPPROTO_ROUTING  ||
2221             next_proto == IPPROTO_FRAGMENT ||
2222             next_proto == IPPROTO_ESP      ||
2223             next_proto == IPPROTO_AH       ||
2224             next_proto == IPPROTO_DSTOPTS)
2225                 return rte_flow_error_set(error, EINVAL,
2226                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2227                                           "IPv6 proto (next header) should "
2228                                           "not be set as extension header");
2229         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2230                 return rte_flow_error_set(error, EINVAL,
2231                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2232                                           "wrong tunnel type - IPv4 specified "
2233                                           "but IPv6 item provided");
2234         if (item_flags & l3m)
2235                 return rte_flow_error_set(error, ENOTSUP,
2236                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2237                                           "multiple L3 layers not supported");
2238         else if (item_flags & l4m)
2239                 return rte_flow_error_set(error, EINVAL,
2240                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2241                                           "L3 cannot follow an L4 layer.");
2242         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2243                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2244                 return rte_flow_error_set(error, EINVAL,
2245                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2246                                           "L3 cannot follow an NVGRE layer.");
2247         if (!mask)
2248                 mask = &rte_flow_item_ipv6_mask;
2249         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2250                                         acc_mask ? (const uint8_t *)acc_mask
2251                                                  : (const uint8_t *)&nic_mask,
2252                                         sizeof(struct rte_flow_item_ipv6),
2253                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2254         if (ret < 0)
2255                 return ret;
2256         return 0;
2257 }
2258
2259 /**
2260  * Validate UDP item.
2261  *
2262  * @param[in] item
2263  *   Item specification.
2264  * @param[in] item_flags
2265  *   Bit-fields that holds the items detected until now.
2266  * @param[in] target_protocol
2267  *   The next protocol in the previous item.
2268  * @param[in] flow_mask
2269  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2270  * @param[out] error
2271  *   Pointer to error structure.
2272  *
2273  * @return
2274  *   0 on success, a negative errno value otherwise and rte_errno is set.
2275  */
2276 int
2277 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2278                             uint64_t item_flags,
2279                             uint8_t target_protocol,
2280                             struct rte_flow_error *error)
2281 {
2282         const struct rte_flow_item_udp *mask = item->mask;
2283         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2284         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2285                                       MLX5_FLOW_LAYER_OUTER_L3;
2286         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2287                                       MLX5_FLOW_LAYER_OUTER_L4;
2288         int ret;
2289
2290         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2291                 return rte_flow_error_set(error, EINVAL,
2292                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2293                                           "protocol filtering not compatible"
2294                                           " with UDP layer");
2295         if (!(item_flags & l3m))
2296                 return rte_flow_error_set(error, EINVAL,
2297                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2298                                           "L3 is mandatory to filter on L4");
2299         if (item_flags & l4m)
2300                 return rte_flow_error_set(error, EINVAL,
2301                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2302                                           "multiple L4 layers not supported");
2303         if (!mask)
2304                 mask = &rte_flow_item_udp_mask;
2305         ret = mlx5_flow_item_acceptable
2306                 (item, (const uint8_t *)mask,
2307                  (const uint8_t *)&rte_flow_item_udp_mask,
2308                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2309                  error);
2310         if (ret < 0)
2311                 return ret;
2312         return 0;
2313 }
2314
2315 /**
2316  * Validate TCP item.
2317  *
2318  * @param[in] item
2319  *   Item specification.
2320  * @param[in] item_flags
2321  *   Bit-fields that holds the items detected until now.
2322  * @param[in] target_protocol
2323  *   The next protocol in the previous item.
2324  * @param[out] error
2325  *   Pointer to error structure.
2326  *
2327  * @return
2328  *   0 on success, a negative errno value otherwise and rte_errno is set.
2329  */
2330 int
2331 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2332                             uint64_t item_flags,
2333                             uint8_t target_protocol,
2334                             const struct rte_flow_item_tcp *flow_mask,
2335                             struct rte_flow_error *error)
2336 {
2337         const struct rte_flow_item_tcp *mask = item->mask;
2338         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2339         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2340                                       MLX5_FLOW_LAYER_OUTER_L3;
2341         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2342                                       MLX5_FLOW_LAYER_OUTER_L4;
2343         int ret;
2344
2345         MLX5_ASSERT(flow_mask);
2346         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2347                 return rte_flow_error_set(error, EINVAL,
2348                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2349                                           "protocol filtering not compatible"
2350                                           " with TCP layer");
2351         if (!(item_flags & l3m))
2352                 return rte_flow_error_set(error, EINVAL,
2353                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2354                                           "L3 is mandatory to filter on L4");
2355         if (item_flags & l4m)
2356                 return rte_flow_error_set(error, EINVAL,
2357                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2358                                           "multiple L4 layers not supported");
2359         if (!mask)
2360                 mask = &rte_flow_item_tcp_mask;
2361         ret = mlx5_flow_item_acceptable
2362                 (item, (const uint8_t *)mask,
2363                  (const uint8_t *)flow_mask,
2364                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2365                  error);
2366         if (ret < 0)
2367                 return ret;
2368         return 0;
2369 }
2370
2371 /**
2372  * Validate VXLAN item.
2373  *
2374  * @param[in] item
2375  *   Item specification.
2376  * @param[in] item_flags
2377  *   Bit-fields that holds the items detected until now.
2378  * @param[in] target_protocol
2379  *   The next protocol in the previous item.
2380  * @param[out] error
2381  *   Pointer to error structure.
2382  *
2383  * @return
2384  *   0 on success, a negative errno value otherwise and rte_errno is set.
2385  */
2386 int
2387 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2388                               uint64_t item_flags,
2389                               struct rte_flow_error *error)
2390 {
2391         const struct rte_flow_item_vxlan *spec = item->spec;
2392         const struct rte_flow_item_vxlan *mask = item->mask;
2393         int ret;
2394         union vni {
2395                 uint32_t vlan_id;
2396                 uint8_t vni[4];
2397         } id = { .vlan_id = 0, };
2398
2399
2400         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2401                 return rte_flow_error_set(error, ENOTSUP,
2402                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2403                                           "multiple tunnel layers not"
2404                                           " supported");
2405         /*
2406          * Verify only UDPv4 is present as defined in
2407          * https://tools.ietf.org/html/rfc7348
2408          */
2409         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2410                 return rte_flow_error_set(error, EINVAL,
2411                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2412                                           "no outer UDP layer found");
2413         if (!mask)
2414                 mask = &rte_flow_item_vxlan_mask;
2415         ret = mlx5_flow_item_acceptable
2416                 (item, (const uint8_t *)mask,
2417                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2418                  sizeof(struct rte_flow_item_vxlan),
2419                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2420         if (ret < 0)
2421                 return ret;
2422         if (spec) {
2423                 memcpy(&id.vni[1], spec->vni, 3);
2424                 memcpy(&id.vni[1], mask->vni, 3);
2425         }
2426         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2427                 return rte_flow_error_set(error, ENOTSUP,
2428                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2429                                           "VXLAN tunnel must be fully defined");
2430         return 0;
2431 }
2432
2433 /**
2434  * Validate VXLAN_GPE item.
2435  *
2436  * @param[in] item
2437  *   Item specification.
2438  * @param[in] item_flags
2439  *   Bit-fields that holds the items detected until now.
2440  * @param[in] priv
2441  *   Pointer to the private data structure.
2442  * @param[in] target_protocol
2443  *   The next protocol in the previous item.
2444  * @param[out] error
2445  *   Pointer to error structure.
2446  *
2447  * @return
2448  *   0 on success, a negative errno value otherwise and rte_errno is set.
2449  */
2450 int
2451 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2452                                   uint64_t item_flags,
2453                                   struct rte_eth_dev *dev,
2454                                   struct rte_flow_error *error)
2455 {
2456         struct mlx5_priv *priv = dev->data->dev_private;
2457         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2458         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2459         int ret;
2460         union vni {
2461                 uint32_t vlan_id;
2462                 uint8_t vni[4];
2463         } id = { .vlan_id = 0, };
2464
2465         if (!priv->config.l3_vxlan_en)
2466                 return rte_flow_error_set(error, ENOTSUP,
2467                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2468                                           "L3 VXLAN is not enabled by device"
2469                                           " parameter and/or not configured in"
2470                                           " firmware");
2471         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2472                 return rte_flow_error_set(error, ENOTSUP,
2473                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2474                                           "multiple tunnel layers not"
2475                                           " supported");
2476         /*
2477          * Verify only UDPv4 is present as defined in
2478          * https://tools.ietf.org/html/rfc7348
2479          */
2480         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2481                 return rte_flow_error_set(error, EINVAL,
2482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2483                                           "no outer UDP layer found");
2484         if (!mask)
2485                 mask = &rte_flow_item_vxlan_gpe_mask;
2486         ret = mlx5_flow_item_acceptable
2487                 (item, (const uint8_t *)mask,
2488                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2489                  sizeof(struct rte_flow_item_vxlan_gpe),
2490                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2491         if (ret < 0)
2492                 return ret;
2493         if (spec) {
2494                 if (spec->protocol)
2495                         return rte_flow_error_set(error, ENOTSUP,
2496                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2497                                                   item,
2498                                                   "VxLAN-GPE protocol"
2499                                                   " not supported");
2500                 memcpy(&id.vni[1], spec->vni, 3);
2501                 memcpy(&id.vni[1], mask->vni, 3);
2502         }
2503         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2504                 return rte_flow_error_set(error, ENOTSUP,
2505                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2506                                           "VXLAN-GPE tunnel must be fully"
2507                                           " defined");
2508         return 0;
2509 }
2510 /**
2511  * Validate GRE Key item.
2512  *
2513  * @param[in] item
2514  *   Item specification.
2515  * @param[in] item_flags
2516  *   Bit flags to mark detected items.
2517  * @param[in] gre_item
2518  *   Pointer to gre_item
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 int
2526 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2527                                 uint64_t item_flags,
2528                                 const struct rte_flow_item *gre_item,
2529                                 struct rte_flow_error *error)
2530 {
2531         const rte_be32_t *mask = item->mask;
2532         int ret = 0;
2533         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2534         const struct rte_flow_item_gre *gre_spec;
2535         const struct rte_flow_item_gre *gre_mask;
2536
2537         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2538                 return rte_flow_error_set(error, ENOTSUP,
2539                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2540                                           "Multiple GRE key not support");
2541         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2542                 return rte_flow_error_set(error, ENOTSUP,
2543                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2544                                           "No preceding GRE header");
2545         if (item_flags & MLX5_FLOW_LAYER_INNER)
2546                 return rte_flow_error_set(error, ENOTSUP,
2547                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2548                                           "GRE key following a wrong item");
2549         gre_mask = gre_item->mask;
2550         if (!gre_mask)
2551                 gre_mask = &rte_flow_item_gre_mask;
2552         gre_spec = gre_item->spec;
2553         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2554                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2555                 return rte_flow_error_set(error, EINVAL,
2556                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2557                                           "Key bit must be on");
2558
2559         if (!mask)
2560                 mask = &gre_key_default_mask;
2561         ret = mlx5_flow_item_acceptable
2562                 (item, (const uint8_t *)mask,
2563                  (const uint8_t *)&gre_key_default_mask,
2564                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2565         return ret;
2566 }
2567
2568 /**
2569  * Validate GRE item.
2570  *
2571  * @param[in] item
2572  *   Item specification.
2573  * @param[in] item_flags
2574  *   Bit flags to mark detected items.
2575  * @param[in] target_protocol
2576  *   The next protocol in the previous item.
2577  * @param[out] error
2578  *   Pointer to error structure.
2579  *
2580  * @return
2581  *   0 on success, a negative errno value otherwise and rte_errno is set.
2582  */
2583 int
2584 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2585                             uint64_t item_flags,
2586                             uint8_t target_protocol,
2587                             struct rte_flow_error *error)
2588 {
2589         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2590         const struct rte_flow_item_gre *mask = item->mask;
2591         int ret;
2592         const struct rte_flow_item_gre nic_mask = {
2593                 .c_rsvd0_ver = RTE_BE16(0xB000),
2594                 .protocol = RTE_BE16(UINT16_MAX),
2595         };
2596
2597         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2598                 return rte_flow_error_set(error, EINVAL,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2600                                           "protocol filtering not compatible"
2601                                           " with this GRE layer");
2602         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2603                 return rte_flow_error_set(error, ENOTSUP,
2604                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2605                                           "multiple tunnel layers not"
2606                                           " supported");
2607         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2608                 return rte_flow_error_set(error, ENOTSUP,
2609                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2610                                           "L3 Layer is missing");
2611         if (!mask)
2612                 mask = &rte_flow_item_gre_mask;
2613         ret = mlx5_flow_item_acceptable
2614                 (item, (const uint8_t *)mask,
2615                  (const uint8_t *)&nic_mask,
2616                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2617                  error);
2618         if (ret < 0)
2619                 return ret;
2620 #ifndef HAVE_MLX5DV_DR
2621 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2622         if (spec && (spec->protocol & mask->protocol))
2623                 return rte_flow_error_set(error, ENOTSUP,
2624                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2625                                           "without MPLS support the"
2626                                           " specification cannot be used for"
2627                                           " filtering");
2628 #endif
2629 #endif
2630         return 0;
2631 }
2632
2633 /**
2634  * Validate Geneve item.
2635  *
2636  * @param[in] item
2637  *   Item specification.
2638  * @param[in] itemFlags
2639  *   Bit-fields that holds the items detected until now.
2640  * @param[in] enPriv
2641  *   Pointer to the private data structure.
2642  * @param[out] error
2643  *   Pointer to error structure.
2644  *
2645  * @return
2646  *   0 on success, a negative errno value otherwise and rte_errno is set.
2647  */
2648
2649 int
2650 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2651                                uint64_t item_flags,
2652                                struct rte_eth_dev *dev,
2653                                struct rte_flow_error *error)
2654 {
2655         struct mlx5_priv *priv = dev->data->dev_private;
2656         const struct rte_flow_item_geneve *spec = item->spec;
2657         const struct rte_flow_item_geneve *mask = item->mask;
2658         int ret;
2659         uint16_t gbhdr;
2660         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2661                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2662         const struct rte_flow_item_geneve nic_mask = {
2663                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2664                 .vni = "\xff\xff\xff",
2665                 .protocol = RTE_BE16(UINT16_MAX),
2666         };
2667
2668         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2669                 return rte_flow_error_set(error, ENOTSUP,
2670                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2671                                           "L3 Geneve is not enabled by device"
2672                                           " parameter and/or not configured in"
2673                                           " firmware");
2674         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2675                 return rte_flow_error_set(error, ENOTSUP,
2676                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2677                                           "multiple tunnel layers not"
2678                                           " supported");
2679         /*
2680          * Verify only UDPv4 is present as defined in
2681          * https://tools.ietf.org/html/rfc7348
2682          */
2683         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2684                 return rte_flow_error_set(error, EINVAL,
2685                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2686                                           "no outer UDP layer found");
2687         if (!mask)
2688                 mask = &rte_flow_item_geneve_mask;
2689         ret = mlx5_flow_item_acceptable
2690                                   (item, (const uint8_t *)mask,
2691                                    (const uint8_t *)&nic_mask,
2692                                    sizeof(struct rte_flow_item_geneve),
2693                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2694         if (ret)
2695                 return ret;
2696         if (spec) {
2697                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2698                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2699                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2700                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2701                         return rte_flow_error_set(error, ENOTSUP,
2702                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2703                                                   item,
2704                                                   "Geneve protocol unsupported"
2705                                                   " fields are being used");
2706                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2707                         return rte_flow_error_set
2708                                         (error, ENOTSUP,
2709                                          RTE_FLOW_ERROR_TYPE_ITEM,
2710                                          item,
2711                                          "Unsupported Geneve options length");
2712         }
2713         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2714                 return rte_flow_error_set
2715                                     (error, ENOTSUP,
2716                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2717                                      "Geneve tunnel must be fully defined");
2718         return 0;
2719 }
2720
2721 /**
2722  * Validate MPLS item.
2723  *
2724  * @param[in] dev
2725  *   Pointer to the rte_eth_dev structure.
2726  * @param[in] item
2727  *   Item specification.
2728  * @param[in] item_flags
2729  *   Bit-fields that holds the items detected until now.
2730  * @param[in] prev_layer
2731  *   The protocol layer indicated in previous item.
2732  * @param[out] error
2733  *   Pointer to error structure.
2734  *
2735  * @return
2736  *   0 on success, a negative errno value otherwise and rte_errno is set.
2737  */
2738 int
2739 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2740                              const struct rte_flow_item *item __rte_unused,
2741                              uint64_t item_flags __rte_unused,
2742                              uint64_t prev_layer __rte_unused,
2743                              struct rte_flow_error *error)
2744 {
2745 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2746         const struct rte_flow_item_mpls *mask = item->mask;
2747         struct mlx5_priv *priv = dev->data->dev_private;
2748         int ret;
2749
2750         if (!priv->config.mpls_en)
2751                 return rte_flow_error_set(error, ENOTSUP,
2752                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2753                                           "MPLS not supported or"
2754                                           " disabled in firmware"
2755                                           " configuration.");
2756         /* MPLS over IP, UDP, GRE is allowed */
2757         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2758                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2759                             MLX5_FLOW_LAYER_GRE)))
2760                 return rte_flow_error_set(error, EINVAL,
2761                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2762                                           "protocol filtering not compatible"
2763                                           " with MPLS layer");
2764         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2765         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2766             !(item_flags & MLX5_FLOW_LAYER_GRE))
2767                 return rte_flow_error_set(error, ENOTSUP,
2768                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2769                                           "multiple tunnel layers not"
2770                                           " supported");
2771         if (!mask)
2772                 mask = &rte_flow_item_mpls_mask;
2773         ret = mlx5_flow_item_acceptable
2774                 (item, (const uint8_t *)mask,
2775                  (const uint8_t *)&rte_flow_item_mpls_mask,
2776                  sizeof(struct rte_flow_item_mpls),
2777                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2778         if (ret < 0)
2779                 return ret;
2780         return 0;
2781 #else
2782         return rte_flow_error_set(error, ENOTSUP,
2783                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2784                                   "MPLS is not supported by Verbs, please"
2785                                   " update.");
2786 #endif
2787 }
2788
2789 /**
2790  * Validate NVGRE item.
2791  *
2792  * @param[in] item
2793  *   Item specification.
2794  * @param[in] item_flags
2795  *   Bit flags to mark detected items.
2796  * @param[in] target_protocol
2797  *   The next protocol in the previous item.
2798  * @param[out] error
2799  *   Pointer to error structure.
2800  *
2801  * @return
2802  *   0 on success, a negative errno value otherwise and rte_errno is set.
2803  */
2804 int
2805 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2806                               uint64_t item_flags,
2807                               uint8_t target_protocol,
2808                               struct rte_flow_error *error)
2809 {
2810         const struct rte_flow_item_nvgre *mask = item->mask;
2811         int ret;
2812
2813         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2814                 return rte_flow_error_set(error, EINVAL,
2815                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2816                                           "protocol filtering not compatible"
2817                                           " with this GRE layer");
2818         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2819                 return rte_flow_error_set(error, ENOTSUP,
2820                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2821                                           "multiple tunnel layers not"
2822                                           " supported");
2823         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2824                 return rte_flow_error_set(error, ENOTSUP,
2825                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2826                                           "L3 Layer is missing");
2827         if (!mask)
2828                 mask = &rte_flow_item_nvgre_mask;
2829         ret = mlx5_flow_item_acceptable
2830                 (item, (const uint8_t *)mask,
2831                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2832                  sizeof(struct rte_flow_item_nvgre),
2833                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2834         if (ret < 0)
2835                 return ret;
2836         return 0;
2837 }
2838
2839 /**
2840  * Validate eCPRI item.
2841  *
2842  * @param[in] item
2843  *   Item specification.
2844  * @param[in] item_flags
2845  *   Bit-fields that holds the items detected until now.
2846  * @param[in] last_item
2847  *   Previous validated item in the pattern items.
2848  * @param[in] ether_type
2849  *   Type in the ethernet layer header (including dot1q).
2850  * @param[in] acc_mask
2851  *   Acceptable mask, if NULL default internal default mask
2852  *   will be used to check whether item fields are supported.
2853  * @param[out] error
2854  *   Pointer to error structure.
2855  *
2856  * @return
2857  *   0 on success, a negative errno value otherwise and rte_errno is set.
2858  */
2859 int
2860 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2861                               uint64_t item_flags,
2862                               uint64_t last_item,
2863                               uint16_t ether_type,
2864                               const struct rte_flow_item_ecpri *acc_mask,
2865                               struct rte_flow_error *error)
2866 {
2867         const struct rte_flow_item_ecpri *mask = item->mask;
2868         const struct rte_flow_item_ecpri nic_mask = {
2869                 .hdr = {
2870                         .common = {
2871                                 .u32 =
2872                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2873                                         .type = 0xFF,
2874                                         }).u32),
2875                         },
2876                         .dummy[0] = 0xFFFFFFFF,
2877                 },
2878         };
2879         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2880                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2881         struct rte_flow_item_ecpri mask_lo;
2882
2883         if ((last_item & outer_l2_vlan) && ether_type &&
2884             ether_type != RTE_ETHER_TYPE_ECPRI)
2885                 return rte_flow_error_set(error, EINVAL,
2886                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2887                                           "eCPRI cannot follow L2/VLAN layer "
2888                                           "which ether type is not 0xAEFE.");
2889         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2890                 return rte_flow_error_set(error, EINVAL,
2891                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2892                                           "eCPRI with tunnel is not supported "
2893                                           "right now.");
2894         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2895                 return rte_flow_error_set(error, ENOTSUP,
2896                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2897                                           "multiple L3 layers not supported");
2898         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2899                 return rte_flow_error_set(error, EINVAL,
2900                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2901                                           "eCPRI cannot follow a TCP layer.");
2902         /* In specification, eCPRI could be over UDP layer. */
2903         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2904                 return rte_flow_error_set(error, EINVAL,
2905                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2906                                           "eCPRI over UDP layer is not yet "
2907                                           "supported right now.");
2908         /* Mask for type field in common header could be zero. */
2909         if (!mask)
2910                 mask = &rte_flow_item_ecpri_mask;
2911         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2912         /* Input mask is in big-endian format. */
2913         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2914                 return rte_flow_error_set(error, EINVAL,
2915                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2916                                           "partial mask is not supported "
2917                                           "for protocol");
2918         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2919                 return rte_flow_error_set(error, EINVAL,
2920                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2921                                           "message header mask must be after "
2922                                           "a type mask");
2923         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2924                                          acc_mask ? (const uint8_t *)acc_mask
2925                                                   : (const uint8_t *)&nic_mask,
2926                                          sizeof(struct rte_flow_item_ecpri),
2927                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2928 }
2929
2930 /* Allocate unique ID for the split Q/RSS subflows. */
2931 static uint32_t
2932 flow_qrss_get_id(struct rte_eth_dev *dev)
2933 {
2934         struct mlx5_priv *priv = dev->data->dev_private;
2935         uint32_t qrss_id, ret;
2936
2937         ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2938         if (ret)
2939                 return 0;
2940         MLX5_ASSERT(qrss_id);
2941         return qrss_id;
2942 }
2943
2944 /* Free unique ID for the split Q/RSS subflows. */
2945 static void
2946 flow_qrss_free_id(struct rte_eth_dev *dev,  uint32_t qrss_id)
2947 {
2948         struct mlx5_priv *priv = dev->data->dev_private;
2949
2950         if (qrss_id)
2951                 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2952 }
2953
2954 /**
2955  * Release resource related QUEUE/RSS action split.
2956  *
2957  * @param dev
2958  *   Pointer to Ethernet device.
2959  * @param flow
2960  *   Flow to release id's from.
2961  */
2962 static void
2963 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2964                              struct rte_flow *flow)
2965 {
2966         struct mlx5_priv *priv = dev->data->dev_private;
2967         uint32_t handle_idx;
2968         struct mlx5_flow_handle *dev_handle;
2969
2970         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2971                        handle_idx, dev_handle, next)
2972                 if (dev_handle->split_flow_id)
2973                         flow_qrss_free_id(dev, dev_handle->split_flow_id);
2974 }
2975
2976 static int
2977 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2978                    const struct rte_flow_attr *attr __rte_unused,
2979                    const struct rte_flow_item items[] __rte_unused,
2980                    const struct rte_flow_action actions[] __rte_unused,
2981                    bool external __rte_unused,
2982                    int hairpin __rte_unused,
2983                    struct rte_flow_error *error)
2984 {
2985         return rte_flow_error_set(error, ENOTSUP,
2986                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2987 }
2988
2989 static struct mlx5_flow *
2990 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2991                   const struct rte_flow_attr *attr __rte_unused,
2992                   const struct rte_flow_item items[] __rte_unused,
2993                   const struct rte_flow_action actions[] __rte_unused,
2994                   struct rte_flow_error *error)
2995 {
2996         rte_flow_error_set(error, ENOTSUP,
2997                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2998         return NULL;
2999 }
3000
3001 static int
3002 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3003                     struct mlx5_flow *dev_flow __rte_unused,
3004                     const struct rte_flow_attr *attr __rte_unused,
3005                     const struct rte_flow_item items[] __rte_unused,
3006                     const struct rte_flow_action actions[] __rte_unused,
3007                     struct rte_flow_error *error)
3008 {
3009         return rte_flow_error_set(error, ENOTSUP,
3010                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3011 }
3012
3013 static int
3014 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3015                 struct rte_flow *flow __rte_unused,
3016                 struct rte_flow_error *error)
3017 {
3018         return rte_flow_error_set(error, ENOTSUP,
3019                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3020 }
3021
3022 static void
3023 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3024                  struct rte_flow *flow __rte_unused)
3025 {
3026 }
3027
3028 static void
3029 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3030                   struct rte_flow *flow __rte_unused)
3031 {
3032 }
3033
3034 static int
3035 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3036                 struct rte_flow *flow __rte_unused,
3037                 const struct rte_flow_action *actions __rte_unused,
3038                 void *data __rte_unused,
3039                 struct rte_flow_error *error)
3040 {
3041         return rte_flow_error_set(error, ENOTSUP,
3042                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3043 }
3044
3045 /* Void driver to protect from null pointer reference. */
3046 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3047         .validate = flow_null_validate,
3048         .prepare = flow_null_prepare,
3049         .translate = flow_null_translate,
3050         .apply = flow_null_apply,
3051         .remove = flow_null_remove,
3052         .destroy = flow_null_destroy,
3053         .query = flow_null_query,
3054 };
3055
3056 /**
3057  * Select flow driver type according to flow attributes and device
3058  * configuration.
3059  *
3060  * @param[in] dev
3061  *   Pointer to the dev structure.
3062  * @param[in] attr
3063  *   Pointer to the flow attributes.
3064  *
3065  * @return
3066  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3067  */
3068 static enum mlx5_flow_drv_type
3069 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3070 {
3071         struct mlx5_priv *priv = dev->data->dev_private;
3072         /* The OS can determine first a specific flow type (DV, VERBS) */
3073         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3074
3075         if (type != MLX5_FLOW_TYPE_MAX)
3076                 return type;
3077         /* If no OS specific type - continue with DV/VERBS selection */
3078         if (attr->transfer && priv->config.dv_esw_en)
3079                 type = MLX5_FLOW_TYPE_DV;
3080         if (!attr->transfer)
3081                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3082                                                  MLX5_FLOW_TYPE_VERBS;
3083         return type;
3084 }
3085
3086 #define flow_get_drv_ops(type) flow_drv_ops[type]
3087
3088 /**
3089  * Flow driver validation API. This abstracts calling driver specific functions.
3090  * The type of flow driver is determined according to flow attributes.
3091  *
3092  * @param[in] dev
3093  *   Pointer to the dev structure.
3094  * @param[in] attr
3095  *   Pointer to the flow attributes.
3096  * @param[in] items
3097  *   Pointer to the list of items.
3098  * @param[in] actions
3099  *   Pointer to the list of actions.
3100  * @param[in] external
3101  *   This flow rule is created by request external to PMD.
3102  * @param[in] hairpin
3103  *   Number of hairpin TX actions, 0 means classic flow.
3104  * @param[out] error
3105  *   Pointer to the error structure.
3106  *
3107  * @return
3108  *   0 on success, a negative errno value otherwise and rte_errno is set.
3109  */
3110 static inline int
3111 flow_drv_validate(struct rte_eth_dev *dev,
3112                   const struct rte_flow_attr *attr,
3113                   const struct rte_flow_item items[],
3114                   const struct rte_flow_action actions[],
3115                   bool external, int hairpin, struct rte_flow_error *error)
3116 {
3117         const struct mlx5_flow_driver_ops *fops;
3118         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3119
3120         fops = flow_get_drv_ops(type);
3121         return fops->validate(dev, attr, items, actions, external,
3122                               hairpin, error);
3123 }
3124
3125 /**
3126  * Flow driver preparation API. This abstracts calling driver specific
3127  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3128  * calculates the size of memory required for device flow, allocates the memory,
3129  * initializes the device flow and returns the pointer.
3130  *
3131  * @note
3132  *   This function initializes device flow structure such as dv or verbs in
3133  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3134  *   rest. For example, adding returning device flow to flow->dev_flow list and
3135  *   setting backward reference to the flow should be done out of this function.
3136  *   layers field is not filled either.
3137  *
3138  * @param[in] dev
3139  *   Pointer to the dev structure.
3140  * @param[in] attr
3141  *   Pointer to the flow attributes.
3142  * @param[in] items
3143  *   Pointer to the list of items.
3144  * @param[in] actions
3145  *   Pointer to the list of actions.
3146  * @param[in] flow_idx
3147  *   This memory pool index to the flow.
3148  * @param[out] error
3149  *   Pointer to the error structure.
3150  *
3151  * @return
3152  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3153  */
3154 static inline struct mlx5_flow *
3155 flow_drv_prepare(struct rte_eth_dev *dev,
3156                  const struct rte_flow *flow,
3157                  const struct rte_flow_attr *attr,
3158                  const struct rte_flow_item items[],
3159                  const struct rte_flow_action actions[],
3160                  uint32_t flow_idx,
3161                  struct rte_flow_error *error)
3162 {
3163         const struct mlx5_flow_driver_ops *fops;
3164         enum mlx5_flow_drv_type type = flow->drv_type;
3165         struct mlx5_flow *mlx5_flow = NULL;
3166
3167         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3168         fops = flow_get_drv_ops(type);
3169         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3170         if (mlx5_flow)
3171                 mlx5_flow->flow_idx = flow_idx;
3172         return mlx5_flow;
3173 }
3174
3175 /**
3176  * Flow driver translation API. This abstracts calling driver specific
3177  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3178  * translates a generic flow into a driver flow. flow_drv_prepare() must
3179  * precede.
3180  *
3181  * @note
3182  *   dev_flow->layers could be filled as a result of parsing during translation
3183  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3184  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3185  *   flow->actions could be overwritten even though all the expanded dev_flows
3186  *   have the same actions.
3187  *
3188  * @param[in] dev
3189  *   Pointer to the rte dev structure.
3190  * @param[in, out] dev_flow
3191  *   Pointer to the mlx5 flow.
3192  * @param[in] attr
3193  *   Pointer to the flow attributes.
3194  * @param[in] items
3195  *   Pointer to the list of items.
3196  * @param[in] actions
3197  *   Pointer to the list of actions.
3198  * @param[out] error
3199  *   Pointer to the error structure.
3200  *
3201  * @return
3202  *   0 on success, a negative errno value otherwise and rte_errno is set.
3203  */
3204 static inline int
3205 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3206                    const struct rte_flow_attr *attr,
3207                    const struct rte_flow_item items[],
3208                    const struct rte_flow_action actions[],
3209                    struct rte_flow_error *error)
3210 {
3211         const struct mlx5_flow_driver_ops *fops;
3212         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3213
3214         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3215         fops = flow_get_drv_ops(type);
3216         return fops->translate(dev, dev_flow, attr, items, actions, error);
3217 }
3218
3219 /**
3220  * Flow driver apply API. This abstracts calling driver specific functions.
3221  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3222  * translated driver flows on to device. flow_drv_translate() must precede.
3223  *
3224  * @param[in] dev
3225  *   Pointer to Ethernet device structure.
3226  * @param[in, out] flow
3227  *   Pointer to flow structure.
3228  * @param[out] error
3229  *   Pointer to error structure.
3230  *
3231  * @return
3232  *   0 on success, a negative errno value otherwise and rte_errno is set.
3233  */
3234 static inline int
3235 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3236                struct rte_flow_error *error)
3237 {
3238         const struct mlx5_flow_driver_ops *fops;
3239         enum mlx5_flow_drv_type type = flow->drv_type;
3240
3241         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3242         fops = flow_get_drv_ops(type);
3243         return fops->apply(dev, flow, error);
3244 }
3245
3246 /**
3247  * Flow driver remove API. This abstracts calling driver specific functions.
3248  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3249  * on device. All the resources of the flow should be freed by calling
3250  * flow_drv_destroy().
3251  *
3252  * @param[in] dev
3253  *   Pointer to Ethernet device.
3254  * @param[in, out] flow
3255  *   Pointer to flow structure.
3256  */
3257 static inline void
3258 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
3259 {
3260         const struct mlx5_flow_driver_ops *fops;
3261         enum mlx5_flow_drv_type type = flow->drv_type;
3262
3263         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3264         fops = flow_get_drv_ops(type);
3265         fops->remove(dev, flow);
3266 }
3267
3268 /**
3269  * Flow driver destroy API. This abstracts calling driver specific functions.
3270  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3271  * on device and releases resources of the flow.
3272  *
3273  * @param[in] dev
3274  *   Pointer to Ethernet device.
3275  * @param[in, out] flow
3276  *   Pointer to flow structure.
3277  */
3278 static inline void
3279 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3280 {
3281         const struct mlx5_flow_driver_ops *fops;
3282         enum mlx5_flow_drv_type type = flow->drv_type;
3283
3284         flow_mreg_split_qrss_release(dev, flow);
3285         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3286         fops = flow_get_drv_ops(type);
3287         fops->destroy(dev, flow);
3288 }
3289
3290 /**
3291  * Get RSS action from the action list.
3292  *
3293  * @param[in] actions
3294  *   Pointer to the list of actions.
3295  *
3296  * @return
3297  *   Pointer to the RSS action if exist, else return NULL.
3298  */
3299 static const struct rte_flow_action_rss*
3300 flow_get_rss_action(const struct rte_flow_action actions[])
3301 {
3302         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3303                 switch (actions->type) {
3304                 case RTE_FLOW_ACTION_TYPE_RSS:
3305                         return (const struct rte_flow_action_rss *)
3306                                actions->conf;
3307                 default:
3308                         break;
3309                 }
3310         }
3311         return NULL;
3312 }
3313
3314 /* maps shared action to translated non shared in some actions array */
3315 struct mlx5_translated_shared_action {
3316         struct rte_flow_shared_action *action; /**< Shared action */
3317         int index; /**< Index in related array of rte_flow_action */
3318 };
3319
3320 /**
3321  * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related
3322  * non shared action if translation possible.
3323  * This functionality used to run same execution path for both shared & non
3324  * shared actions on flow create. All necessary preparations for shared
3325  * action handling should be preformed on *shared* actions list returned
3326  * from this call.
3327  *
3328  * @param[in] actions
3329  *   List of actions to translate.
3330  * @param[out] shared
3331  *   List to store translated shared actions.
3332  * @param[in, out] shared_n
3333  *   Size of *shared* array. On return should be updated with number of shared
3334  *   actions retrieved from the *actions* list.
3335  * @param[out] translated_actions
3336  *   List of actions where all shared actions were translated to non shared
3337  *   if possible. NULL if no translation took place.
3338  * @param[out] error
3339  *   Pointer to the error structure.
3340  *
3341  * @return
3342  *   0 on success, a negative errno value otherwise and rte_errno is set.
3343  */
3344 static int
3345 flow_shared_actions_translate(const struct rte_flow_action actions[],
3346         struct mlx5_translated_shared_action *shared,
3347         int *shared_n,
3348         struct rte_flow_action **translated_actions,
3349         struct rte_flow_error *error)
3350 {
3351         struct rte_flow_action *translated = NULL;
3352         size_t actions_size;
3353         int n;
3354         int copied_n = 0;
3355         struct mlx5_translated_shared_action *shared_end = NULL;
3356
3357         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3358                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED)
3359                         continue;
3360                 if (copied_n == *shared_n) {
3361                         return rte_flow_error_set
3362                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3363                                  NULL, "too many shared actions");
3364                 }
3365                 rte_memcpy(&shared[copied_n].action, &actions[n].conf,
3366                            sizeof(actions[n].conf));
3367                 shared[copied_n].index = n;
3368                 copied_n++;
3369         }
3370         n++;
3371         *shared_n = copied_n;
3372         if (!copied_n)
3373                 return 0;
3374         actions_size = sizeof(struct rte_flow_action) * n;
3375         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3376         if (!translated) {
3377                 rte_errno = ENOMEM;
3378                 return -ENOMEM;
3379         }
3380         memcpy(translated, actions, actions_size);
3381         for (shared_end = shared + copied_n; shared < shared_end; shared++) {
3382                 const struct rte_flow_shared_action *shared_action;
3383
3384                 shared_action = shared->action;
3385                 switch (shared_action->type) {
3386                 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
3387                         translated[shared->index].type =
3388                                 RTE_FLOW_ACTION_TYPE_RSS;
3389                         translated[shared->index].conf =
3390                                 &shared_action->rss.origin;
3391                         break;
3392                 default:
3393                         mlx5_free(translated);
3394                         return rte_flow_error_set
3395                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3396                                  NULL, "invalid shared action type");
3397                 }
3398         }
3399         *translated_actions = translated;
3400         return 0;
3401 }
3402
3403 /**
3404  * Get Shared RSS action from the action list.
3405  *
3406  * @param[in] shared
3407  *   Pointer to the list of actions.
3408  * @param[in] shared_n
3409  *   Actions list length.
3410  *
3411  * @return
3412  *   Pointer to the MLX5 RSS action if exists, otherwise return NULL.
3413  */
3414 static struct mlx5_shared_action_rss *
3415 flow_get_shared_rss_action(struct mlx5_translated_shared_action *shared,
3416                            int shared_n)
3417 {
3418         struct mlx5_translated_shared_action *shared_end;
3419
3420         for (shared_end = shared + shared_n; shared < shared_end; shared++) {
3421                 struct rte_flow_shared_action *shared_action;
3422
3423                 shared_action = shared->action;
3424                 switch (shared_action->type) {
3425                 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
3426                         __atomic_add_fetch(&shared_action->refcnt, 1,
3427                                            __ATOMIC_RELAXED);
3428                         return &shared_action->rss;
3429                 default:
3430                         break;
3431                 }
3432         }
3433         return NULL;
3434 }
3435
3436 struct rte_flow_shared_action *
3437 mlx5_flow_get_shared_rss(struct rte_flow *flow)
3438 {
3439         if (flow->shared_rss)
3440                 return container_of(flow->shared_rss,
3441                                     struct rte_flow_shared_action, rss);
3442         else
3443                 return NULL;
3444 }
3445
3446 static unsigned int
3447 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3448 {
3449         const struct rte_flow_item *item;
3450         unsigned int has_vlan = 0;
3451
3452         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3453                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3454                         has_vlan = 1;
3455                         break;
3456                 }
3457         }
3458         if (has_vlan)
3459                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3460                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3461         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3462                                MLX5_EXPANSION_ROOT_OUTER;
3463 }
3464
3465 /**
3466  *  Get layer flags from the prefix flow.
3467  *
3468  *  Some flows may be split to several subflows, the prefix subflow gets the
3469  *  match items and the suffix sub flow gets the actions.
3470  *  Some actions need the user defined match item flags to get the detail for
3471  *  the action.
3472  *  This function helps the suffix flow to get the item layer flags from prefix
3473  *  subflow.
3474  *
3475  * @param[in] dev_flow
3476  *   Pointer the created preifx subflow.
3477  *
3478  * @return
3479  *   The layers get from prefix subflow.
3480  */
3481 static inline uint64_t
3482 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3483 {
3484         uint64_t layers = 0;
3485
3486         /*
3487          * Layers bits could be localization, but usually the compiler will
3488          * help to do the optimization work for source code.
3489          * If no decap actions, use the layers directly.
3490          */
3491         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3492                 return dev_flow->handle->layers;
3493         /* Convert L3 layers with decap action. */
3494         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3495                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3496         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3497                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3498         /* Convert L4 layers with decap action.  */
3499         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3500                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3501         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3502                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3503         return layers;
3504 }
3505
3506 /**
3507  * Get metadata split action information.
3508  *
3509  * @param[in] actions
3510  *   Pointer to the list of actions.
3511  * @param[out] qrss
3512  *   Pointer to the return pointer.
3513  * @param[out] qrss_type
3514  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3515  *   if no QUEUE/RSS is found.
3516  * @param[out] encap_idx
3517  *   Pointer to the index of the encap action if exists, otherwise the last
3518  *   action index.
3519  *
3520  * @return
3521  *   Total number of actions.
3522  */
3523 static int
3524 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3525                                        const struct rte_flow_action **qrss,
3526                                        int *encap_idx)
3527 {
3528         const struct rte_flow_action_raw_encap *raw_encap;
3529         int actions_n = 0;
3530         int raw_decap_idx = -1;
3531
3532         *encap_idx = -1;
3533         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3534                 switch (actions->type) {
3535                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3536                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3537                         *encap_idx = actions_n;
3538                         break;
3539                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3540                         raw_decap_idx = actions_n;
3541                         break;
3542                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3543                         raw_encap = actions->conf;
3544                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3545                                 *encap_idx = raw_decap_idx != -1 ?
3546                                                       raw_decap_idx : actions_n;
3547                         break;
3548                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3549                 case RTE_FLOW_ACTION_TYPE_RSS:
3550                         *qrss = actions;
3551                         break;
3552                 default:
3553                         break;
3554                 }
3555                 actions_n++;
3556         }
3557         if (*encap_idx == -1)
3558                 *encap_idx = actions_n;
3559         /* Count RTE_FLOW_ACTION_TYPE_END. */
3560         return actions_n + 1;
3561 }
3562
3563 /**
3564  * Check meter action from the action list.
3565  *
3566  * @param[in] actions
3567  *   Pointer to the list of actions.
3568  * @param[out] mtr
3569  *   Pointer to the meter exist flag.
3570  *
3571  * @return
3572  *   Total number of actions.
3573  */
3574 static int
3575 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3576 {
3577         int actions_n = 0;
3578
3579         MLX5_ASSERT(mtr);
3580         *mtr = 0;
3581         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3582                 switch (actions->type) {
3583                 case RTE_FLOW_ACTION_TYPE_METER:
3584                         *mtr = 1;
3585                         break;
3586                 default:
3587                         break;
3588                 }
3589                 actions_n++;
3590         }
3591         /* Count RTE_FLOW_ACTION_TYPE_END. */
3592         return actions_n + 1;
3593 }
3594
3595 /**
3596  * Check if the flow should be split due to hairpin.
3597  * The reason for the split is that in current HW we can't
3598  * support encap and push-vlan on Rx, so if a flow contains
3599  * these actions we move it to Tx.
3600  *
3601  * @param dev
3602  *   Pointer to Ethernet device.
3603  * @param[in] attr
3604  *   Flow rule attributes.
3605  * @param[in] actions
3606  *   Associated actions (list terminated by the END action).
3607  *
3608  * @return
3609  *   > 0 the number of actions and the flow should be split,
3610  *   0 when no split required.
3611  */
3612 static int
3613 flow_check_hairpin_split(struct rte_eth_dev *dev,
3614                          const struct rte_flow_attr *attr,
3615                          const struct rte_flow_action actions[])
3616 {
3617         int queue_action = 0;
3618         int action_n = 0;
3619         int split = 0;
3620         const struct rte_flow_action_queue *queue;
3621         const struct rte_flow_action_rss *rss;
3622         const struct rte_flow_action_raw_encap *raw_encap;
3623         const struct rte_eth_hairpin_conf *conf;
3624
3625         if (!attr->ingress)
3626                 return 0;
3627         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3628                 switch (actions->type) {
3629                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3630                         queue = actions->conf;
3631                         if (queue == NULL)
3632                                 return 0;
3633                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3634                         if (conf != NULL && !!conf->tx_explicit)
3635                                 return 0;
3636                         queue_action = 1;
3637                         action_n++;
3638                         break;
3639                 case RTE_FLOW_ACTION_TYPE_RSS:
3640                         rss = actions->conf;
3641                         if (rss == NULL || rss->queue_num == 0)
3642                                 return 0;
3643                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3644                         if (conf != NULL && !!conf->tx_explicit)
3645                                 return 0;
3646                         queue_action = 1;
3647                         action_n++;
3648                         break;
3649                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3650                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3651                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3652                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3653                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3654                         split++;
3655                         action_n++;
3656                         break;
3657                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3658                         raw_encap = actions->conf;
3659                         if (raw_encap->size >
3660                             (sizeof(struct rte_flow_item_eth) +
3661                              sizeof(struct rte_flow_item_ipv4)))
3662                                 split++;
3663                         action_n++;
3664                         break;
3665                 default:
3666                         action_n++;
3667                         break;
3668                 }
3669         }
3670         if (split && queue_action)
3671                 return action_n;
3672         return 0;
3673 }
3674
3675 /* Declare flow create/destroy prototype in advance. */
3676 static uint32_t
3677 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3678                  const struct rte_flow_attr *attr,
3679                  const struct rte_flow_item items[],
3680                  const struct rte_flow_action actions[],
3681                  bool external, struct rte_flow_error *error);
3682
3683 static void
3684 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3685                   uint32_t flow_idx);
3686
3687 /**
3688  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3689  *
3690  * As mark_id is unique, if there's already a registered flow for the mark_id,
3691  * return by increasing the reference counter of the resource. Otherwise, create
3692  * the resource (mcp_res) and flow.
3693  *
3694  * Flow looks like,
3695  *   - If ingress port is ANY and reg_c[1] is mark_id,
3696  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3697  *
3698  * For default flow (zero mark_id), flow is like,
3699  *   - If ingress port is ANY,
3700  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3701  *
3702  * @param dev
3703  *   Pointer to Ethernet device.
3704  * @param mark_id
3705  *   ID of MARK action, zero means default flow for META.
3706  * @param[out] error
3707  *   Perform verbose error reporting if not NULL.
3708  *
3709  * @return
3710  *   Associated resource on success, NULL otherwise and rte_errno is set.
3711  */
3712 static struct mlx5_flow_mreg_copy_resource *
3713 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3714                           struct rte_flow_error *error)
3715 {
3716         struct mlx5_priv *priv = dev->data->dev_private;
3717         struct rte_flow_attr attr = {
3718                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3719                 .ingress = 1,
3720         };
3721         struct mlx5_rte_flow_item_tag tag_spec = {
3722                 .data = mark_id,
3723         };
3724         struct rte_flow_item items[] = {
3725                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3726         };
3727         struct rte_flow_action_mark ftag = {
3728                 .id = mark_id,
3729         };
3730         struct mlx5_flow_action_copy_mreg cp_mreg = {
3731                 .dst = REG_B,
3732                 .src = REG_NON,
3733         };
3734         struct rte_flow_action_jump jump = {
3735                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3736         };
3737         struct rte_flow_action actions[] = {
3738                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3739         };
3740         struct mlx5_flow_mreg_copy_resource *mcp_res;
3741         uint32_t idx = 0;
3742         int ret;
3743
3744         /* Fill the register fileds in the flow. */
3745         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3746         if (ret < 0)
3747                 return NULL;
3748         tag_spec.id = ret;
3749         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3750         if (ret < 0)
3751                 return NULL;
3752         cp_mreg.src = ret;
3753         /* Check if already registered. */
3754         MLX5_ASSERT(priv->mreg_cp_tbl);
3755         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
3756         if (mcp_res) {
3757                 /* For non-default rule. */
3758                 if (mark_id != MLX5_DEFAULT_COPY_ID)
3759                         mcp_res->refcnt++;
3760                 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
3761                             mcp_res->refcnt == 1);
3762                 return mcp_res;
3763         }
3764         /* Provide the full width of FLAG specific value. */
3765         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3766                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3767         /* Build a new flow. */
3768         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3769                 items[0] = (struct rte_flow_item){
3770                         .type = (enum rte_flow_item_type)
3771                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3772                         .spec = &tag_spec,
3773                 };
3774                 items[1] = (struct rte_flow_item){
3775                         .type = RTE_FLOW_ITEM_TYPE_END,
3776                 };
3777                 actions[0] = (struct rte_flow_action){
3778                         .type = (enum rte_flow_action_type)
3779                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3780                         .conf = &ftag,
3781                 };
3782                 actions[1] = (struct rte_flow_action){
3783                         .type = (enum rte_flow_action_type)
3784                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3785                         .conf = &cp_mreg,
3786                 };
3787                 actions[2] = (struct rte_flow_action){
3788                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3789                         .conf = &jump,
3790                 };
3791                 actions[3] = (struct rte_flow_action){
3792                         .type = RTE_FLOW_ACTION_TYPE_END,
3793                 };
3794         } else {
3795                 /* Default rule, wildcard match. */
3796                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3797                 items[0] = (struct rte_flow_item){
3798                         .type = RTE_FLOW_ITEM_TYPE_END,
3799                 };
3800                 actions[0] = (struct rte_flow_action){
3801                         .type = (enum rte_flow_action_type)
3802                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3803                         .conf = &cp_mreg,
3804                 };
3805                 actions[1] = (struct rte_flow_action){
3806                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3807                         .conf = &jump,
3808                 };
3809                 actions[2] = (struct rte_flow_action){
3810                         .type = RTE_FLOW_ACTION_TYPE_END,
3811                 };
3812         }
3813         /* Build a new entry. */
3814         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3815         if (!mcp_res) {
3816                 rte_errno = ENOMEM;
3817                 return NULL;
3818         }
3819         mcp_res->idx = idx;
3820         /*
3821          * The copy Flows are not included in any list. There
3822          * ones are referenced from other Flows and can not
3823          * be applied, removed, deleted in ardbitrary order
3824          * by list traversing.
3825          */
3826         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3827                                          actions, false, error);
3828         if (!mcp_res->rix_flow)
3829                 goto error;
3830         mcp_res->refcnt++;
3831         mcp_res->hlist_ent.key = mark_id;
3832         ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3833                                 &mcp_res->hlist_ent);
3834         MLX5_ASSERT(!ret);
3835         if (ret)
3836                 goto error;
3837         return mcp_res;
3838 error:
3839         if (mcp_res->rix_flow)
3840                 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3841         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3842         return NULL;
3843 }
3844
3845 /**
3846  * Release flow in RX_CP_TBL.
3847  *
3848  * @param dev
3849  *   Pointer to Ethernet device.
3850  * @flow
3851  *   Parent flow for wich copying is provided.
3852  */
3853 static void
3854 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3855                           struct rte_flow *flow)
3856 {
3857         struct mlx5_flow_mreg_copy_resource *mcp_res;
3858         struct mlx5_priv *priv = dev->data->dev_private;
3859
3860         if (!flow->rix_mreg_copy)
3861                 return;
3862         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3863                                  flow->rix_mreg_copy);
3864         if (!mcp_res || !priv->mreg_cp_tbl)
3865                 return;
3866         if (flow->copy_applied) {
3867                 MLX5_ASSERT(mcp_res->appcnt);
3868                 flow->copy_applied = 0;
3869                 --mcp_res->appcnt;
3870                 if (!mcp_res->appcnt) {
3871                         struct rte_flow *mcp_flow = mlx5_ipool_get
3872                                         (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3873                                         mcp_res->rix_flow);
3874
3875                         if (mcp_flow)
3876                                 flow_drv_remove(dev, mcp_flow);
3877                 }
3878         }
3879         /*
3880          * We do not check availability of metadata registers here,
3881          * because copy resources are not allocated in this case.
3882          */
3883         if (--mcp_res->refcnt)
3884                 return;
3885         MLX5_ASSERT(mcp_res->rix_flow);
3886         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3887         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3888         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3889         flow->rix_mreg_copy = 0;
3890 }
3891
3892 /**
3893  * Start flow in RX_CP_TBL.
3894  *
3895  * @param dev
3896  *   Pointer to Ethernet device.
3897  * @flow
3898  *   Parent flow for wich copying is provided.
3899  *
3900  * @return
3901  *   0 on success, a negative errno value otherwise and rte_errno is set.
3902  */
3903 static int
3904 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3905                             struct rte_flow *flow)
3906 {
3907         struct mlx5_flow_mreg_copy_resource *mcp_res;
3908         struct mlx5_priv *priv = dev->data->dev_private;
3909         int ret;
3910
3911         if (!flow->rix_mreg_copy || flow->copy_applied)
3912                 return 0;
3913         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3914                                  flow->rix_mreg_copy);
3915         if (!mcp_res)
3916                 return 0;
3917         if (!mcp_res->appcnt) {
3918                 struct rte_flow *mcp_flow = mlx5_ipool_get
3919                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3920                                 mcp_res->rix_flow);
3921
3922                 if (mcp_flow) {
3923                         ret = flow_drv_apply(dev, mcp_flow, NULL);
3924                         if (ret)
3925                                 return ret;
3926                 }
3927         }
3928         ++mcp_res->appcnt;
3929         flow->copy_applied = 1;
3930         return 0;
3931 }
3932
3933 /**
3934  * Stop flow in RX_CP_TBL.
3935  *
3936  * @param dev
3937  *   Pointer to Ethernet device.
3938  * @flow
3939  *   Parent flow for wich copying is provided.
3940  */
3941 static void
3942 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3943                            struct rte_flow *flow)
3944 {
3945         struct mlx5_flow_mreg_copy_resource *mcp_res;
3946         struct mlx5_priv *priv = dev->data->dev_private;
3947
3948         if (!flow->rix_mreg_copy || !flow->copy_applied)
3949                 return;
3950         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3951                                  flow->rix_mreg_copy);
3952         if (!mcp_res)
3953                 return;
3954         MLX5_ASSERT(mcp_res->appcnt);
3955         --mcp_res->appcnt;
3956         flow->copy_applied = 0;
3957         if (!mcp_res->appcnt) {
3958                 struct rte_flow *mcp_flow = mlx5_ipool_get
3959                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3960                                 mcp_res->rix_flow);
3961
3962                 if (mcp_flow)
3963                         flow_drv_remove(dev, mcp_flow);
3964         }
3965 }
3966
3967 /**
3968  * Remove the default copy action from RX_CP_TBL.
3969  *
3970  * @param dev
3971  *   Pointer to Ethernet device.
3972  */
3973 static void
3974 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3975 {
3976         struct mlx5_flow_mreg_copy_resource *mcp_res;
3977         struct mlx5_priv *priv = dev->data->dev_private;
3978
3979         /* Check if default flow is registered. */
3980         if (!priv->mreg_cp_tbl)
3981                 return;
3982         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3983                                             MLX5_DEFAULT_COPY_ID);
3984         if (!mcp_res)
3985                 return;
3986         MLX5_ASSERT(mcp_res->rix_flow);
3987         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3988         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3989         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3990 }
3991
3992 /**
3993  * Add the default copy action in in RX_CP_TBL.
3994  *
3995  * @param dev
3996  *   Pointer to Ethernet device.
3997  * @param[out] error
3998  *   Perform verbose error reporting if not NULL.
3999  *
4000  * @return
4001  *   0 for success, negative value otherwise and rte_errno is set.
4002  */
4003 static int
4004 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4005                                   struct rte_flow_error *error)
4006 {
4007         struct mlx5_priv *priv = dev->data->dev_private;
4008         struct mlx5_flow_mreg_copy_resource *mcp_res;
4009
4010         /* Check whether extensive metadata feature is engaged. */
4011         if (!priv->config.dv_flow_en ||
4012             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4013             !mlx5_flow_ext_mreg_supported(dev) ||
4014             !priv->sh->dv_regc0_mask)
4015                 return 0;
4016         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4017         if (!mcp_res)
4018                 return -rte_errno;
4019         return 0;
4020 }
4021
4022 /**
4023  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4024  *
4025  * All the flow having Q/RSS action should be split by
4026  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4027  * performs the following,
4028  *   - CQE->flow_tag := reg_c[1] (MARK)
4029  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4030  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4031  * but there should be a flow per each MARK ID set by MARK action.
4032  *
4033  * For the aforementioned reason, if there's a MARK action in flow's action
4034  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4035  * the MARK ID to CQE's flow_tag like,
4036  *   - If reg_c[1] is mark_id,
4037  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4038  *
4039  * For SET_META action which stores value in reg_c[0], as the destination is
4040  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4041  * MARK ID means the default flow. The default flow looks like,
4042  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4043  *
4044  * @param dev
4045  *   Pointer to Ethernet device.
4046  * @param flow
4047  *   Pointer to flow structure.
4048  * @param[in] actions
4049  *   Pointer to the list of actions.
4050  * @param[out] error
4051  *   Perform verbose error reporting if not NULL.
4052  *
4053  * @return
4054  *   0 on success, negative value otherwise and rte_errno is set.
4055  */
4056 static int
4057 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4058                             struct rte_flow *flow,
4059                             const struct rte_flow_action *actions,
4060                             struct rte_flow_error *error)
4061 {
4062         struct mlx5_priv *priv = dev->data->dev_private;
4063         struct mlx5_dev_config *config = &priv->config;
4064         struct mlx5_flow_mreg_copy_resource *mcp_res;
4065         const struct rte_flow_action_mark *mark;
4066
4067         /* Check whether extensive metadata feature is engaged. */
4068         if (!config->dv_flow_en ||
4069             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4070             !mlx5_flow_ext_mreg_supported(dev) ||
4071             !priv->sh->dv_regc0_mask)
4072                 return 0;
4073         /* Find MARK action. */
4074         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4075                 switch (actions->type) {
4076                 case RTE_FLOW_ACTION_TYPE_FLAG:
4077                         mcp_res = flow_mreg_add_copy_action
4078                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4079                         if (!mcp_res)
4080                                 return -rte_errno;
4081                         flow->rix_mreg_copy = mcp_res->idx;
4082                         if (dev->data->dev_started) {
4083                                 mcp_res->appcnt++;
4084                                 flow->copy_applied = 1;
4085                         }
4086                         return 0;
4087                 case RTE_FLOW_ACTION_TYPE_MARK:
4088                         mark = (const struct rte_flow_action_mark *)
4089                                 actions->conf;
4090                         mcp_res =
4091                                 flow_mreg_add_copy_action(dev, mark->id, error);
4092                         if (!mcp_res)
4093                                 return -rte_errno;
4094                         flow->rix_mreg_copy = mcp_res->idx;
4095                         if (dev->data->dev_started) {
4096                                 mcp_res->appcnt++;
4097                                 flow->copy_applied = 1;
4098                         }
4099                         return 0;
4100                 default:
4101                         break;
4102                 }
4103         }
4104         return 0;
4105 }
4106
4107 #define MLX5_MAX_SPLIT_ACTIONS 24
4108 #define MLX5_MAX_SPLIT_ITEMS 24
4109
4110 /**
4111  * Split the hairpin flow.
4112  * Since HW can't support encap and push-vlan on Rx, we move these
4113  * actions to Tx.
4114  * If the count action is after the encap then we also
4115  * move the count action. in this case the count will also measure
4116  * the outer bytes.
4117  *
4118  * @param dev
4119  *   Pointer to Ethernet device.
4120  * @param[in] actions
4121  *   Associated actions (list terminated by the END action).
4122  * @param[out] actions_rx
4123  *   Rx flow actions.
4124  * @param[out] actions_tx
4125  *   Tx flow actions..
4126  * @param[out] pattern_tx
4127  *   The pattern items for the Tx flow.
4128  * @param[out] flow_id
4129  *   The flow ID connected to this flow.
4130  *
4131  * @return
4132  *   0 on success.
4133  */
4134 static int
4135 flow_hairpin_split(struct rte_eth_dev *dev,
4136                    const struct rte_flow_action actions[],
4137                    struct rte_flow_action actions_rx[],
4138                    struct rte_flow_action actions_tx[],
4139                    struct rte_flow_item pattern_tx[],
4140                    uint32_t *flow_id)
4141 {
4142         struct mlx5_priv *priv = dev->data->dev_private;
4143         const struct rte_flow_action_raw_encap *raw_encap;
4144         const struct rte_flow_action_raw_decap *raw_decap;
4145         struct mlx5_rte_flow_action_set_tag *set_tag;
4146         struct rte_flow_action *tag_action;
4147         struct mlx5_rte_flow_item_tag *tag_item;
4148         struct rte_flow_item *item;
4149         char *addr;
4150         int encap = 0;
4151
4152         mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
4153         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4154                 switch (actions->type) {
4155                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4156                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4157                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4158                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4159                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4160                         rte_memcpy(actions_tx, actions,
4161                                sizeof(struct rte_flow_action));
4162                         actions_tx++;
4163                         break;
4164                 case RTE_FLOW_ACTION_TYPE_COUNT:
4165                         if (encap) {
4166                                 rte_memcpy(actions_tx, actions,
4167                                            sizeof(struct rte_flow_action));
4168                                 actions_tx++;
4169                         } else {
4170                                 rte_memcpy(actions_rx, actions,
4171                                            sizeof(struct rte_flow_action));
4172                                 actions_rx++;
4173                         }
4174                         break;
4175                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4176                         raw_encap = actions->conf;
4177                         if (raw_encap->size >
4178                             (sizeof(struct rte_flow_item_eth) +
4179                              sizeof(struct rte_flow_item_ipv4))) {
4180                                 memcpy(actions_tx, actions,
4181                                        sizeof(struct rte_flow_action));
4182                                 actions_tx++;
4183                                 encap = 1;
4184                         } else {
4185                                 rte_memcpy(actions_rx, actions,
4186                                            sizeof(struct rte_flow_action));
4187                                 actions_rx++;
4188                         }
4189                         break;
4190                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4191                         raw_decap = actions->conf;
4192                         if (raw_decap->size <
4193                             (sizeof(struct rte_flow_item_eth) +
4194                              sizeof(struct rte_flow_item_ipv4))) {
4195                                 memcpy(actions_tx, actions,
4196                                        sizeof(struct rte_flow_action));
4197                                 actions_tx++;
4198                         } else {
4199                                 rte_memcpy(actions_rx, actions,
4200                                            sizeof(struct rte_flow_action));
4201                                 actions_rx++;
4202                         }
4203                         break;
4204                 default:
4205                         rte_memcpy(actions_rx, actions,
4206                                    sizeof(struct rte_flow_action));
4207                         actions_rx++;
4208                         break;
4209                 }
4210         }
4211         /* Add set meta action and end action for the Rx flow. */
4212         tag_action = actions_rx;
4213         tag_action->type = (enum rte_flow_action_type)
4214                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4215         actions_rx++;
4216         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4217         actions_rx++;
4218         set_tag = (void *)actions_rx;
4219         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
4220         MLX5_ASSERT(set_tag->id > REG_NON);
4221         set_tag->data = *flow_id;
4222         tag_action->conf = set_tag;
4223         /* Create Tx item list. */
4224         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4225         addr = (void *)&pattern_tx[2];
4226         item = pattern_tx;
4227         item->type = (enum rte_flow_item_type)
4228                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4229         tag_item = (void *)addr;
4230         tag_item->data = *flow_id;
4231         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4232         MLX5_ASSERT(set_tag->id > REG_NON);
4233         item->spec = tag_item;
4234         addr += sizeof(struct mlx5_rte_flow_item_tag);
4235         tag_item = (void *)addr;
4236         tag_item->data = UINT32_MAX;
4237         tag_item->id = UINT16_MAX;
4238         item->mask = tag_item;
4239         item->last = NULL;
4240         item++;
4241         item->type = RTE_FLOW_ITEM_TYPE_END;
4242         return 0;
4243 }
4244
4245 __extension__
4246 union tunnel_offload_mark {
4247         uint32_t val;
4248         struct {
4249                 uint32_t app_reserve:8;
4250                 uint32_t table_id:15;
4251                 uint32_t transfer:1;
4252                 uint32_t _unused_:8;
4253         };
4254 };
4255
4256 struct tunnel_default_miss_ctx {
4257         uint16_t *queue;
4258         __extension__
4259         union {
4260                 struct rte_flow_action_rss action_rss;
4261                 struct rte_flow_action_queue miss_queue;
4262                 struct rte_flow_action_jump miss_jump;
4263                 uint8_t raw[0];
4264         };
4265 };
4266
4267 static int
4268 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
4269                              struct rte_flow *flow,
4270                              const struct rte_flow_attr *attr,
4271                              const struct rte_flow_action *app_actions,
4272                              uint32_t flow_idx,
4273                              struct tunnel_default_miss_ctx *ctx,
4274                              struct rte_flow_error *error)
4275 {
4276         struct mlx5_priv *priv = dev->data->dev_private;
4277         struct mlx5_flow *dev_flow;
4278         struct rte_flow_attr miss_attr = *attr;
4279         const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf;
4280         const struct rte_flow_item miss_items[2] = {
4281                 {
4282                         .type = RTE_FLOW_ITEM_TYPE_ETH,
4283                         .spec = NULL,
4284                         .last = NULL,
4285                         .mask = NULL
4286                 },
4287                 {
4288                         .type = RTE_FLOW_ITEM_TYPE_END,
4289                         .spec = NULL,
4290                         .last = NULL,
4291                         .mask = NULL
4292                 }
4293         };
4294         union tunnel_offload_mark mark_id;
4295         struct rte_flow_action_mark miss_mark;
4296         struct rte_flow_action miss_actions[3] = {
4297                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
4298                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
4299         };
4300         const struct rte_flow_action_jump *jump_data;
4301         uint32_t i, flow_table = 0; /* prevent compilation warning */
4302         struct flow_grp_info grp_info = {
4303                 .external = 1,
4304                 .transfer = attr->transfer,
4305                 .fdb_def_rule = !!priv->fdb_def_rule,
4306                 .std_tbl_fix = 0,
4307         };
4308         int ret;
4309
4310         if (!attr->transfer) {
4311                 uint32_t q_size;
4312
4313                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
4314                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
4315                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
4316                                          0, SOCKET_ID_ANY);
4317                 if (!ctx->queue)
4318                         return rte_flow_error_set
4319                                 (error, ENOMEM,
4320                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4321                                 NULL, "invalid default miss RSS");
4322                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
4323                 ctx->action_rss.level = 0,
4324                 ctx->action_rss.types = priv->rss_conf.rss_hf,
4325                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
4326                 ctx->action_rss.queue_num = priv->reta_idx_n,
4327                 ctx->action_rss.key = priv->rss_conf.rss_key,
4328                 ctx->action_rss.queue = ctx->queue;
4329                 if (!priv->reta_idx_n || !priv->rxqs_n)
4330                         return rte_flow_error_set
4331                                 (error, EINVAL,
4332                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4333                                 NULL, "invalid port configuration");
4334                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
4335                         ctx->action_rss.types = 0;
4336                 for (i = 0; i != priv->reta_idx_n; ++i)
4337                         ctx->queue[i] = (*priv->reta_idx)[i];
4338         } else {
4339                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
4340                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
4341         }
4342         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
4343         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
4344         jump_data = app_actions->conf;
4345         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
4346         miss_attr.group = jump_data->group;
4347         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
4348                                        &flow_table, grp_info, error);
4349         if (ret)
4350                 return rte_flow_error_set(error, EINVAL,
4351                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4352                                           NULL, "invalid tunnel id");
4353         mark_id.app_reserve = 0;
4354         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
4355         mark_id.transfer = !!attr->transfer;
4356         mark_id._unused_ = 0;
4357         miss_mark.id = mark_id.val;
4358         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
4359                                     miss_items, miss_actions, flow_idx, error);
4360         if (!dev_flow)
4361                 return -rte_errno;
4362         dev_flow->flow = flow;
4363         dev_flow->external = true;
4364         dev_flow->tunnel = tunnel;
4365         /* Subflow object was created, we must include one in the list. */
4366         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4367                       dev_flow->handle, next);
4368         DRV_LOG(DEBUG,
4369                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
4370                 dev->data->port_id, tunnel->app_tunnel.type,
4371                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
4372         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
4373                                   miss_actions, error);
4374         if (!ret)
4375                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
4376                                                   error);
4377
4378         return ret;
4379 }
4380
4381 /**
4382  * The last stage of splitting chain, just creates the subflow
4383  * without any modification.
4384  *
4385  * @param[in] dev
4386  *   Pointer to Ethernet device.
4387  * @param[in] flow
4388  *   Parent flow structure pointer.
4389  * @param[in, out] sub_flow
4390  *   Pointer to return the created subflow, may be NULL.
4391  * @param[in] prefix_layers
4392  *   Prefix subflow layers, may be 0.
4393  * @param[in] prefix_mark
4394  *   Prefix subflow mark flag, may be 0.
4395  * @param[in] attr
4396  *   Flow rule attributes.
4397  * @param[in] items
4398  *   Pattern specification (list terminated by the END pattern item).
4399  * @param[in] actions
4400  *   Associated actions (list terminated by the END action).
4401  * @param[in] external
4402  *   This flow rule is created by request external to PMD.
4403  * @param[in] flow_idx
4404  *   This memory pool index to the flow.
4405  * @param[out] error
4406  *   Perform verbose error reporting if not NULL.
4407  * @return
4408  *   0 on success, negative value otherwise
4409  */
4410 static int
4411 flow_create_split_inner(struct rte_eth_dev *dev,
4412                         struct rte_flow *flow,
4413                         struct mlx5_flow **sub_flow,
4414                         uint64_t prefix_layers,
4415                         uint32_t prefix_mark,
4416                         const struct rte_flow_attr *attr,
4417                         const struct rte_flow_item items[],
4418                         const struct rte_flow_action actions[],
4419                         bool external, uint32_t flow_idx,
4420                         struct rte_flow_error *error)
4421 {
4422         struct mlx5_flow *dev_flow;
4423
4424         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4425                 flow_idx, error);
4426         if (!dev_flow)
4427                 return -rte_errno;
4428         dev_flow->flow = flow;
4429         dev_flow->external = external;
4430         /* Subflow object was created, we must include one in the list. */
4431         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4432                       dev_flow->handle, next);
4433         /*
4434          * If dev_flow is as one of the suffix flow, some actions in suffix
4435          * flow may need some user defined item layer flags, and pass the
4436          * Metadate rxq mark flag to suffix flow as well.
4437          */
4438         if (prefix_layers)
4439                 dev_flow->handle->layers = prefix_layers;
4440         if (prefix_mark)
4441                 dev_flow->handle->mark = 1;
4442         if (sub_flow)
4443                 *sub_flow = dev_flow;
4444         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4445 }
4446
4447 /**
4448  * Split the meter flow.
4449  *
4450  * As meter flow will split to three sub flow, other than meter
4451  * action, the other actions make sense to only meter accepts
4452  * the packet. If it need to be dropped, no other additional
4453  * actions should be take.
4454  *
4455  * One kind of special action which decapsulates the L3 tunnel
4456  * header will be in the prefix sub flow, as not to take the
4457  * L3 tunnel header into account.
4458  *
4459  * @param dev
4460  *   Pointer to Ethernet device.
4461  * @param[in] items
4462  *   Pattern specification (list terminated by the END pattern item).
4463  * @param[out] sfx_items
4464  *   Suffix flow match items (list terminated by the END pattern item).
4465  * @param[in] actions
4466  *   Associated actions (list terminated by the END action).
4467  * @param[out] actions_sfx
4468  *   Suffix flow actions.
4469  * @param[out] actions_pre
4470  *   Prefix flow actions.
4471  * @param[out] pattern_sfx
4472  *   The pattern items for the suffix flow.
4473  * @param[out] tag_sfx
4474  *   Pointer to suffix flow tag.
4475  *
4476  * @return
4477  *   0 on success.
4478  */
4479 static int
4480 flow_meter_split_prep(struct rte_eth_dev *dev,
4481                  const struct rte_flow_item items[],
4482                  struct rte_flow_item sfx_items[],
4483                  const struct rte_flow_action actions[],
4484                  struct rte_flow_action actions_sfx[],
4485                  struct rte_flow_action actions_pre[])
4486 {
4487         struct rte_flow_action *tag_action = NULL;
4488         struct rte_flow_item *tag_item;
4489         struct mlx5_rte_flow_action_set_tag *set_tag;
4490         struct rte_flow_error error;
4491         const struct rte_flow_action_raw_encap *raw_encap;
4492         const struct rte_flow_action_raw_decap *raw_decap;
4493         struct mlx5_rte_flow_item_tag *tag_spec;
4494         struct mlx5_rte_flow_item_tag *tag_mask;
4495         uint32_t tag_id;
4496         bool copy_vlan = false;
4497
4498         /* Prepare the actions for prefix and suffix flow. */
4499         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4500                 struct rte_flow_action **action_cur = NULL;
4501
4502                 switch (actions->type) {
4503                 case RTE_FLOW_ACTION_TYPE_METER:
4504                         /* Add the extra tag action first. */
4505                         tag_action = actions_pre;
4506                         tag_action->type = (enum rte_flow_action_type)
4507                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4508                         actions_pre++;
4509                         action_cur = &actions_pre;
4510                         break;
4511                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4512                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4513                         action_cur = &actions_pre;
4514                         break;
4515                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4516                         raw_encap = actions->conf;
4517                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4518                                 action_cur = &actions_pre;
4519                         break;
4520                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4521                         raw_decap = actions->conf;
4522                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4523                                 action_cur = &actions_pre;
4524                         break;
4525                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4526                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4527                         copy_vlan = true;
4528                         break;
4529                 default:
4530                         break;
4531                 }
4532                 if (!action_cur)
4533                         action_cur = &actions_sfx;
4534                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4535                 (*action_cur)++;
4536         }
4537         /* Add end action to the actions. */
4538         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4539         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4540         actions_pre++;
4541         /* Set the tag. */
4542         set_tag = (void *)actions_pre;
4543         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4544         /*
4545          * Get the id from the qrss_pool to make qrss share the id with meter.
4546          */
4547         tag_id = flow_qrss_get_id(dev);
4548         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4549         assert(tag_action);
4550         tag_action->conf = set_tag;
4551         /* Prepare the suffix subflow items. */
4552         tag_item = sfx_items++;
4553         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4554                 int item_type = items->type;
4555
4556                 switch (item_type) {
4557                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4558                         memcpy(sfx_items, items, sizeof(*sfx_items));
4559                         sfx_items++;
4560                         break;
4561                 case RTE_FLOW_ITEM_TYPE_VLAN:
4562                         if (copy_vlan) {
4563                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4564                                 /*
4565                                  * Convert to internal match item, it is used
4566                                  * for vlan push and set vid.
4567                                  */
4568                                 sfx_items->type = (enum rte_flow_item_type)
4569                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4570                                 sfx_items++;
4571                         }
4572                         break;
4573                 default:
4574                         break;
4575                 }
4576         }
4577         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4578         sfx_items++;
4579         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4580         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4581         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4582         tag_mask = tag_spec + 1;
4583         tag_mask->data = 0xffffff00;
4584         tag_item->type = (enum rte_flow_item_type)
4585                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4586         tag_item->spec = tag_spec;
4587         tag_item->last = NULL;
4588         tag_item->mask = tag_mask;
4589         return tag_id;
4590 }
4591
4592 /**
4593  * Split action list having QUEUE/RSS for metadata register copy.
4594  *
4595  * Once Q/RSS action is detected in user's action list, the flow action
4596  * should be split in order to copy metadata registers, which will happen in
4597  * RX_CP_TBL like,
4598  *   - CQE->flow_tag := reg_c[1] (MARK)
4599  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4600  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4601  * This is because the last action of each flow must be a terminal action
4602  * (QUEUE, RSS or DROP).
4603  *
4604  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4605  * stored and kept in the mlx5_flow structure per each sub_flow.
4606  *
4607  * The Q/RSS action is replaced with,
4608  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4609  * And the following JUMP action is added at the end,
4610  *   - JUMP, to RX_CP_TBL.
4611  *
4612  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4613  * flow_create_split_metadata() routine. The flow will look like,
4614  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4615  *
4616  * @param dev
4617  *   Pointer to Ethernet device.
4618  * @param[out] split_actions
4619  *   Pointer to store split actions to jump to CP_TBL.
4620  * @param[in] actions
4621  *   Pointer to the list of original flow actions.
4622  * @param[in] qrss
4623  *   Pointer to the Q/RSS action.
4624  * @param[in] actions_n
4625  *   Number of original actions.
4626  * @param[out] error
4627  *   Perform verbose error reporting if not NULL.
4628  *
4629  * @return
4630  *   non-zero unique flow_id on success, otherwise 0 and
4631  *   error/rte_error are set.
4632  */
4633 static uint32_t
4634 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4635                           struct rte_flow_action *split_actions,
4636                           const struct rte_flow_action *actions,
4637                           const struct rte_flow_action *qrss,
4638                           int actions_n, struct rte_flow_error *error)
4639 {
4640         struct mlx5_rte_flow_action_set_tag *set_tag;
4641         struct rte_flow_action_jump *jump;
4642         const int qrss_idx = qrss - actions;
4643         uint32_t flow_id = 0;
4644         int ret = 0;
4645
4646         /*
4647          * Given actions will be split
4648          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4649          * - Add jump to mreg CP_TBL.
4650          * As a result, there will be one more action.
4651          */
4652         ++actions_n;
4653         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4654         set_tag = (void *)(split_actions + actions_n);
4655         /*
4656          * If tag action is not set to void(it means we are not the meter
4657          * suffix flow), add the tag action. Since meter suffix flow already
4658          * has the tag added.
4659          */
4660         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4661                 /*
4662                  * Allocate the new subflow ID. This one is unique within
4663                  * device and not shared with representors. Otherwise,
4664                  * we would have to resolve multi-thread access synch
4665                  * issue. Each flow on the shared device is appended
4666                  * with source vport identifier, so the resulting
4667                  * flows will be unique in the shared (by master and
4668                  * representors) domain even if they have coinciding
4669                  * IDs.
4670                  */
4671                 flow_id = flow_qrss_get_id(dev);
4672                 if (!flow_id)
4673                         return rte_flow_error_set(error, ENOMEM,
4674                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4675                                                   NULL, "can't allocate id "
4676                                                   "for split Q/RSS subflow");
4677                 /* Internal SET_TAG action to set flow ID. */
4678                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4679                         .data = flow_id,
4680                 };
4681                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4682                 if (ret < 0)
4683                         return ret;
4684                 set_tag->id = ret;
4685                 /* Construct new actions array. */
4686                 /* Replace QUEUE/RSS action. */
4687                 split_actions[qrss_idx] = (struct rte_flow_action){
4688                         .type = (enum rte_flow_action_type)
4689                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4690                         .conf = set_tag,
4691                 };
4692         }
4693         /* JUMP action to jump to mreg copy table (CP_TBL). */
4694         jump = (void *)(set_tag + 1);
4695         *jump = (struct rte_flow_action_jump){
4696                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4697         };
4698         split_actions[actions_n - 2] = (struct rte_flow_action){
4699                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4700                 .conf = jump,
4701         };
4702         split_actions[actions_n - 1] = (struct rte_flow_action){
4703                 .type = RTE_FLOW_ACTION_TYPE_END,
4704         };
4705         return flow_id;
4706 }
4707
4708 /**
4709  * Extend the given action list for Tx metadata copy.
4710  *
4711  * Copy the given action list to the ext_actions and add flow metadata register
4712  * copy action in order to copy reg_a set by WQE to reg_c[0].
4713  *
4714  * @param[out] ext_actions
4715  *   Pointer to the extended action list.
4716  * @param[in] actions
4717  *   Pointer to the list of actions.
4718  * @param[in] actions_n
4719  *   Number of actions in the list.
4720  * @param[out] error
4721  *   Perform verbose error reporting if not NULL.
4722  * @param[in] encap_idx
4723  *   The encap action inndex.
4724  *
4725  * @return
4726  *   0 on success, negative value otherwise
4727  */
4728 static int
4729 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4730                        struct rte_flow_action *ext_actions,
4731                        const struct rte_flow_action *actions,
4732                        int actions_n, struct rte_flow_error *error,
4733                        int encap_idx)
4734 {
4735         struct mlx5_flow_action_copy_mreg *cp_mreg =
4736                 (struct mlx5_flow_action_copy_mreg *)
4737                         (ext_actions + actions_n + 1);
4738         int ret;
4739
4740         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4741         if (ret < 0)
4742                 return ret;
4743         cp_mreg->dst = ret;
4744         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4745         if (ret < 0)
4746                 return ret;
4747         cp_mreg->src = ret;
4748         if (encap_idx != 0)
4749                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4750         if (encap_idx == actions_n - 1) {
4751                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4752                         .type = (enum rte_flow_action_type)
4753                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4754                         .conf = cp_mreg,
4755                 };
4756                 ext_actions[actions_n] = (struct rte_flow_action){
4757                         .type = RTE_FLOW_ACTION_TYPE_END,
4758                 };
4759         } else {
4760                 ext_actions[encap_idx] = (struct rte_flow_action){
4761                         .type = (enum rte_flow_action_type)
4762                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4763                         .conf = cp_mreg,
4764                 };
4765                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4766                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4767         }
4768         return 0;
4769 }
4770
4771 /**
4772  * Check the match action from the action list.
4773  *
4774  * @param[in] actions
4775  *   Pointer to the list of actions.
4776  * @param[in] attr
4777  *   Flow rule attributes.
4778  * @param[in] action
4779  *   The action to be check if exist.
4780  * @param[out] match_action_pos
4781  *   Pointer to the position of the matched action if exists, otherwise is -1.
4782  * @param[out] qrss_action_pos
4783  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
4784  *
4785  * @return
4786  *   > 0 the total number of actions.
4787  *   0 if not found match action in action list.
4788  */
4789 static int
4790 flow_check_match_action(const struct rte_flow_action actions[],
4791                         const struct rte_flow_attr *attr,
4792                         enum rte_flow_action_type action,
4793                         int *match_action_pos, int *qrss_action_pos)
4794 {
4795         const struct rte_flow_action_sample *sample;
4796         int actions_n = 0;
4797         int jump_flag = 0;
4798         uint32_t ratio = 0;
4799         int sub_type = 0;
4800         int flag = 0;
4801
4802         *match_action_pos = -1;
4803         *qrss_action_pos = -1;
4804         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4805                 if (actions->type == action) {
4806                         flag = 1;
4807                         *match_action_pos = actions_n;
4808                 }
4809                 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE ||
4810                     actions->type == RTE_FLOW_ACTION_TYPE_RSS)
4811                         *qrss_action_pos = actions_n;
4812                 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP)
4813                         jump_flag = 1;
4814                 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
4815                         sample = actions->conf;
4816                         ratio = sample->ratio;
4817                         sub_type = ((const struct rte_flow_action *)
4818                                         (sample->actions))->type;
4819                 }
4820                 actions_n++;
4821         }
4822         if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) {
4823                 if (ratio == 1) {
4824                         /* JUMP Action not support for Mirroring;
4825                          * Mirroring support multi-destination;
4826                          */
4827                         if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END)
4828                                 flag = 0;
4829                 }
4830         }
4831         /* Count RTE_FLOW_ACTION_TYPE_END. */
4832         return flag ? actions_n + 1 : 0;
4833 }
4834
4835 #define SAMPLE_SUFFIX_ITEM 2
4836
4837 /**
4838  * Split the sample flow.
4839  *
4840  * As sample flow will split to two sub flow, sample flow with
4841  * sample action, the other actions will move to new suffix flow.
4842  *
4843  * Also add unique tag id with tag action in the sample flow,
4844  * the same tag id will be as match in the suffix flow.
4845  *
4846  * @param dev
4847  *   Pointer to Ethernet device.
4848  * @param[in] fdb_tx
4849  *   FDB egress flow flag.
4850  * @param[out] sfx_items
4851  *   Suffix flow match items (list terminated by the END pattern item).
4852  * @param[in] actions
4853  *   Associated actions (list terminated by the END action).
4854  * @param[out] actions_sfx
4855  *   Suffix flow actions.
4856  * @param[out] actions_pre
4857  *   Prefix flow actions.
4858  * @param[in] actions_n
4859  *  The total number of actions.
4860  * @param[in] sample_action_pos
4861  *   The sample action position.
4862  * @param[in] qrss_action_pos
4863  *   The Queue/RSS action position.
4864  * @param[out] error
4865  *   Perform verbose error reporting if not NULL.
4866  *
4867  * @return
4868  *   0 on success, or unique flow_id, a negative errno value
4869  *   otherwise and rte_errno is set.
4870  */
4871 static int
4872 flow_sample_split_prep(struct rte_eth_dev *dev,
4873                        uint32_t fdb_tx,
4874                        struct rte_flow_item sfx_items[],
4875                        const struct rte_flow_action actions[],
4876                        struct rte_flow_action actions_sfx[],
4877                        struct rte_flow_action actions_pre[],
4878                        int actions_n,
4879                        int sample_action_pos,
4880                        int qrss_action_pos,
4881                        struct rte_flow_error *error)
4882 {
4883         struct mlx5_rte_flow_action_set_tag *set_tag;
4884         struct mlx5_rte_flow_item_tag *tag_spec;
4885         struct mlx5_rte_flow_item_tag *tag_mask;
4886         uint32_t tag_id = 0;
4887         int index;
4888         int ret;
4889
4890         if (sample_action_pos < 0)
4891                 return rte_flow_error_set(error, EINVAL,
4892                                           RTE_FLOW_ERROR_TYPE_ACTION,
4893                                           NULL, "invalid position of sample "
4894                                           "action in list");
4895         if (!fdb_tx) {
4896                 /* Prepare the prefix tag action. */
4897                 set_tag = (void *)(actions_pre + actions_n + 1);
4898                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
4899                 if (ret < 0)
4900                         return ret;
4901                 set_tag->id = ret;
4902                 tag_id = flow_qrss_get_id(dev);
4903                 set_tag->data = tag_id;
4904                 /* Prepare the suffix subflow items. */
4905                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
4906                 tag_spec->data = tag_id;
4907                 tag_spec->id = set_tag->id;
4908                 tag_mask = tag_spec + 1;
4909                 tag_mask->data = UINT32_MAX;
4910                 sfx_items[0] = (struct rte_flow_item){
4911                         .type = (enum rte_flow_item_type)
4912                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4913                         .spec = tag_spec,
4914                         .last = NULL,
4915                         .mask = tag_mask,
4916                 };
4917                 sfx_items[1] = (struct rte_flow_item){
4918                         .type = (enum rte_flow_item_type)
4919                                 RTE_FLOW_ITEM_TYPE_END,
4920                 };
4921         }
4922         /* Prepare the actions for prefix and suffix flow. */
4923         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
4924                 index = qrss_action_pos;
4925                 /* Put the preceding the Queue/RSS action into prefix flow. */
4926                 if (index != 0)
4927                         memcpy(actions_pre, actions,
4928                                sizeof(struct rte_flow_action) * index);
4929                 /* Put others preceding the sample action into prefix flow. */
4930                 if (sample_action_pos > index + 1)
4931                         memcpy(actions_pre + index, actions + index + 1,
4932                                sizeof(struct rte_flow_action) *
4933                                (sample_action_pos - index - 1));
4934                 index = sample_action_pos - 1;
4935                 /* Put Queue/RSS action into Suffix flow. */
4936                 memcpy(actions_sfx, actions + qrss_action_pos,
4937                        sizeof(struct rte_flow_action));
4938                 actions_sfx++;
4939         } else {
4940                 index = sample_action_pos;
4941                 if (index != 0)
4942                         memcpy(actions_pre, actions,
4943                                sizeof(struct rte_flow_action) * index);
4944         }
4945         /* Add the extra tag action for NIC-RX and E-Switch ingress. */
4946         if (!fdb_tx) {
4947                 actions_pre[index++] =
4948                         (struct rte_flow_action){
4949                         .type = (enum rte_flow_action_type)
4950                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4951                         .conf = set_tag,
4952                 };
4953         }
4954         memcpy(actions_pre + index, actions + sample_action_pos,
4955                sizeof(struct rte_flow_action));
4956         index += 1;
4957         actions_pre[index] = (struct rte_flow_action){
4958                 .type = (enum rte_flow_action_type)
4959                         RTE_FLOW_ACTION_TYPE_END,
4960         };
4961         /* Put the actions after sample into Suffix flow. */
4962         memcpy(actions_sfx, actions + sample_action_pos + 1,
4963                sizeof(struct rte_flow_action) *
4964                (actions_n - sample_action_pos - 1));
4965         return tag_id;
4966 }
4967
4968 /**
4969  * The splitting for metadata feature.
4970  *
4971  * - Q/RSS action on NIC Rx should be split in order to pass by
4972  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4973  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4974  *
4975  * - All the actions on NIC Tx should have a mreg copy action to
4976  *   copy reg_a from WQE to reg_c[0].
4977  *
4978  * @param dev
4979  *   Pointer to Ethernet device.
4980  * @param[in] flow
4981  *   Parent flow structure pointer.
4982  * @param[in] prefix_layers
4983  *   Prefix flow layer flags.
4984  * @param[in] prefix_mark
4985  *   Prefix subflow mark flag, may be 0.
4986  * @param[in] attr
4987  *   Flow rule attributes.
4988  * @param[in] items
4989  *   Pattern specification (list terminated by the END pattern item).
4990  * @param[in] actions
4991  *   Associated actions (list terminated by the END action).
4992  * @param[in] external
4993  *   This flow rule is created by request external to PMD.
4994  * @param[in] flow_idx
4995  *   This memory pool index to the flow.
4996  * @param[out] error
4997  *   Perform verbose error reporting if not NULL.
4998  * @return
4999  *   0 on success, negative value otherwise
5000  */
5001 static int
5002 flow_create_split_metadata(struct rte_eth_dev *dev,
5003                            struct rte_flow *flow,
5004                            uint64_t prefix_layers,
5005                            uint32_t prefix_mark,
5006                            const struct rte_flow_attr *attr,
5007                            const struct rte_flow_item items[],
5008                            const struct rte_flow_action actions[],
5009                            bool external, uint32_t flow_idx,
5010                            struct rte_flow_error *error)
5011 {
5012         struct mlx5_priv *priv = dev->data->dev_private;
5013         struct mlx5_dev_config *config = &priv->config;
5014         const struct rte_flow_action *qrss = NULL;
5015         struct rte_flow_action *ext_actions = NULL;
5016         struct mlx5_flow *dev_flow = NULL;
5017         uint32_t qrss_id = 0;
5018         int mtr_sfx = 0;
5019         size_t act_size;
5020         int actions_n;
5021         int encap_idx;
5022         int ret;
5023
5024         /* Check whether extensive metadata feature is engaged. */
5025         if (!config->dv_flow_en ||
5026             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5027             !mlx5_flow_ext_mreg_supported(dev))
5028                 return flow_create_split_inner(dev, flow, NULL, prefix_layers,
5029                                                prefix_mark, attr, items,
5030                                                actions, external, flow_idx,
5031                                                error);
5032         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5033                                                            &encap_idx);
5034         if (qrss) {
5035                 /* Exclude hairpin flows from splitting. */
5036                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5037                         const struct rte_flow_action_queue *queue;
5038
5039                         queue = qrss->conf;
5040                         if (mlx5_rxq_get_type(dev, queue->index) ==
5041                             MLX5_RXQ_TYPE_HAIRPIN)
5042                                 qrss = NULL;
5043                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5044                         const struct rte_flow_action_rss *rss;
5045
5046                         rss = qrss->conf;
5047                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5048                             MLX5_RXQ_TYPE_HAIRPIN)
5049                                 qrss = NULL;
5050                 }
5051         }
5052         if (qrss) {
5053                 /* Check if it is in meter suffix table. */
5054                 mtr_sfx = attr->group == (attr->transfer ?
5055                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
5056                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
5057                 /*
5058                  * Q/RSS action on NIC Rx should be split in order to pass by
5059                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5060                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5061                  */
5062                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5063                            sizeof(struct rte_flow_action_set_tag) +
5064                            sizeof(struct rte_flow_action_jump);
5065                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5066                                           SOCKET_ID_ANY);
5067                 if (!ext_actions)
5068                         return rte_flow_error_set(error, ENOMEM,
5069                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5070                                                   NULL, "no memory to split "
5071                                                   "metadata flow");
5072                 /*
5073                  * If we are the suffix flow of meter, tag already exist.
5074                  * Set the tag action to void.
5075                  */
5076                 if (mtr_sfx)
5077                         ext_actions[qrss - actions].type =
5078                                                 RTE_FLOW_ACTION_TYPE_VOID;
5079                 else
5080                         ext_actions[qrss - actions].type =
5081                                                 (enum rte_flow_action_type)
5082                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5083                 /*
5084                  * Create the new actions list with removed Q/RSS action
5085                  * and appended set tag and jump to register copy table
5086                  * (RX_CP_TBL). We should preallocate unique tag ID here
5087                  * in advance, because it is needed for set tag action.
5088                  */
5089                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5090                                                     qrss, actions_n, error);
5091                 if (!mtr_sfx && !qrss_id) {
5092                         ret = -rte_errno;
5093                         goto exit;
5094                 }
5095         } else if (attr->egress && !attr->transfer) {
5096                 /*
5097                  * All the actions on NIC Tx should have a metadata register
5098                  * copy action to copy reg_a from WQE to reg_c[meta]
5099                  */
5100                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5101                            sizeof(struct mlx5_flow_action_copy_mreg);
5102                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5103                                           SOCKET_ID_ANY);
5104                 if (!ext_actions)
5105                         return rte_flow_error_set(error, ENOMEM,
5106                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5107                                                   NULL, "no memory to split "
5108                                                   "metadata flow");
5109                 /* Create the action list appended with copy register. */
5110                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5111                                              actions_n, error, encap_idx);
5112                 if (ret < 0)
5113                         goto exit;
5114         }
5115         /* Add the unmodified original or prefix subflow. */
5116         ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers,
5117                                       prefix_mark, attr,
5118                                       items, ext_actions ? ext_actions :
5119                                       actions, external, flow_idx, error);
5120         if (ret < 0)
5121                 goto exit;
5122         MLX5_ASSERT(dev_flow);
5123         if (qrss) {
5124                 const struct rte_flow_attr q_attr = {
5125                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5126                         .ingress = 1,
5127                 };
5128                 /* Internal PMD action to set register. */
5129                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5130                         .data = qrss_id,
5131                         .id = REG_NON,
5132                 };
5133                 struct rte_flow_item q_items[] = {
5134                         {
5135                                 .type = (enum rte_flow_item_type)
5136                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5137                                 .spec = &q_tag_spec,
5138                                 .last = NULL,
5139                                 .mask = NULL,
5140                         },
5141                         {
5142                                 .type = RTE_FLOW_ITEM_TYPE_END,
5143                         },
5144                 };
5145                 struct rte_flow_action q_actions[] = {
5146                         {
5147                                 .type = qrss->type,
5148                                 .conf = qrss->conf,
5149                         },
5150                         {
5151                                 .type = RTE_FLOW_ACTION_TYPE_END,
5152                         },
5153                 };
5154                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5155
5156                 /*
5157                  * Configure the tag item only if there is no meter subflow.
5158                  * Since tag is already marked in the meter suffix subflow
5159                  * we can just use the meter suffix items as is.
5160                  */
5161                 if (qrss_id) {
5162                         /* Not meter subflow. */
5163                         MLX5_ASSERT(!mtr_sfx);
5164                         /*
5165                          * Put unique id in prefix flow due to it is destroyed
5166                          * after suffix flow and id will be freed after there
5167                          * is no actual flows with this id and identifier
5168                          * reallocation becomes possible (for example, for
5169                          * other flows in other threads).
5170                          */
5171                         dev_flow->handle->split_flow_id = qrss_id;
5172                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5173                                                    error);
5174                         if (ret < 0)
5175                                 goto exit;
5176                         q_tag_spec.id = ret;
5177                 }
5178                 dev_flow = NULL;
5179                 /* Add suffix subflow to execute Q/RSS. */
5180                 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0,
5181                                               &q_attr, mtr_sfx ? items :
5182                                               q_items, q_actions,
5183                                               external, flow_idx, error);
5184                 if (ret < 0)
5185                         goto exit;
5186                 /* qrss ID should be freed if failed. */
5187                 qrss_id = 0;
5188                 MLX5_ASSERT(dev_flow);
5189         }
5190
5191 exit:
5192         /*
5193          * We do not destroy the partially created sub_flows in case of error.
5194          * These ones are included into parent flow list and will be destroyed
5195          * by flow_drv_destroy.
5196          */
5197         flow_qrss_free_id(dev, qrss_id);
5198         mlx5_free(ext_actions);
5199         return ret;
5200 }
5201
5202 /**
5203  * The splitting for meter feature.
5204  *
5205  * - The meter flow will be split to two flows as prefix and
5206  *   suffix flow. The packets make sense only it pass the prefix
5207  *   meter action.
5208  *
5209  * - Reg_C_5 is used for the packet to match betweend prefix and
5210  *   suffix flow.
5211  *
5212  * @param dev
5213  *   Pointer to Ethernet device.
5214  * @param[in] flow
5215  *   Parent flow structure pointer.
5216  * @param[in] prefix_layers
5217  *   Prefix subflow layers, may be 0.
5218  * @param[in] prefix_mark
5219  *   Prefix subflow mark flag, may be 0.
5220  * @param[in] attr
5221  *   Flow rule attributes.
5222  * @param[in] items
5223  *   Pattern specification (list terminated by the END pattern item).
5224  * @param[in] actions
5225  *   Associated actions (list terminated by the END action).
5226  * @param[in] external
5227  *   This flow rule is created by request external to PMD.
5228  * @param[in] flow_idx
5229  *   This memory pool index to the flow.
5230  * @param[out] error
5231  *   Perform verbose error reporting if not NULL.
5232  * @return
5233  *   0 on success, negative value otherwise
5234  */
5235 static int
5236 flow_create_split_meter(struct rte_eth_dev *dev,
5237                         struct rte_flow *flow,
5238                         uint64_t prefix_layers,
5239                         uint32_t prefix_mark,
5240                         const struct rte_flow_attr *attr,
5241                         const struct rte_flow_item items[],
5242                         const struct rte_flow_action actions[],
5243                         bool external, uint32_t flow_idx,
5244                         struct rte_flow_error *error)
5245 {
5246         struct mlx5_priv *priv = dev->data->dev_private;
5247         struct rte_flow_action *sfx_actions = NULL;
5248         struct rte_flow_action *pre_actions = NULL;
5249         struct rte_flow_item *sfx_items = NULL;
5250         struct mlx5_flow *dev_flow = NULL;
5251         struct rte_flow_attr sfx_attr = *attr;
5252         uint32_t mtr = 0;
5253         uint32_t mtr_tag_id = 0;
5254         size_t act_size;
5255         size_t item_size;
5256         int actions_n = 0;
5257         int ret;
5258
5259         if (priv->mtr_en)
5260                 actions_n = flow_check_meter_action(actions, &mtr);
5261         if (mtr) {
5262                 /* The five prefix actions: meter, decap, encap, tag, end. */
5263                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
5264                            sizeof(struct mlx5_rte_flow_action_set_tag);
5265                 /* tag, vlan, port id, end. */
5266 #define METER_SUFFIX_ITEM 4
5267                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5268                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5269                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5270                                           0, SOCKET_ID_ANY);
5271                 if (!sfx_actions)
5272                         return rte_flow_error_set(error, ENOMEM,
5273                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5274                                                   NULL, "no memory to split "
5275                                                   "meter flow");
5276                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5277                              act_size);
5278                 pre_actions = sfx_actions + actions_n;
5279                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
5280                                                    actions, sfx_actions,
5281                                                    pre_actions);
5282                 if (!mtr_tag_id) {
5283                         ret = -rte_errno;
5284                         goto exit;
5285                 }
5286                 /* Add the prefix subflow. */
5287                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5288                                               prefix_layers, 0,
5289                                               attr, items,
5290                                               pre_actions, external,
5291                                               flow_idx, error);
5292                 if (ret) {
5293                         ret = -rte_errno;
5294                         goto exit;
5295                 }
5296                 dev_flow->handle->split_flow_id = mtr_tag_id;
5297                 /* Setting the sfx group atrr. */
5298                 sfx_attr.group = sfx_attr.transfer ?
5299                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
5300                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
5301         }
5302         /* Add the prefix subflow. */
5303         ret = flow_create_split_metadata(dev, flow, dev_flow ?
5304                                          flow_get_prefix_layer_flags(dev_flow) :
5305                                          prefix_layers, dev_flow ?
5306                                          dev_flow->handle->mark : prefix_mark,
5307                                          &sfx_attr, sfx_items ?
5308                                          sfx_items : items,
5309                                          sfx_actions ? sfx_actions : actions,
5310                                          external, flow_idx, error);
5311 exit:
5312         if (sfx_actions)
5313                 mlx5_free(sfx_actions);
5314         return ret;
5315 }
5316
5317 /**
5318  * The splitting for sample feature.
5319  *
5320  * Once Sample action is detected in the action list, the flow actions should
5321  * be split into prefix sub flow and suffix sub flow.
5322  *
5323  * The original items remain in the prefix sub flow, all actions preceding the
5324  * sample action and the sample action itself will be copied to the prefix
5325  * sub flow, the actions following the sample action will be copied to the
5326  * suffix sub flow, Queue action always be located in the suffix sub flow.
5327  *
5328  * In order to make the packet from prefix sub flow matches with suffix sub
5329  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5330  * flow uses tag item with the unique flow id.
5331  *
5332  * @param dev
5333  *   Pointer to Ethernet device.
5334  * @param[in] flow
5335  *   Parent flow structure pointer.
5336  * @param[in] attr
5337  *   Flow rule attributes.
5338  * @param[in] items
5339  *   Pattern specification (list terminated by the END pattern item).
5340  * @param[in] actions
5341  *   Associated actions (list terminated by the END action).
5342  * @param[in] external
5343  *   This flow rule is created by request external to PMD.
5344  * @param[in] flow_idx
5345  *   This memory pool index to the flow.
5346  * @param[out] error
5347  *   Perform verbose error reporting if not NULL.
5348  * @return
5349  *   0 on success, negative value otherwise
5350  */
5351 static int
5352 flow_create_split_sample(struct rte_eth_dev *dev,
5353                          struct rte_flow *flow,
5354                          const struct rte_flow_attr *attr,
5355                          const struct rte_flow_item items[],
5356                          const struct rte_flow_action actions[],
5357                          bool external, uint32_t flow_idx,
5358                          struct rte_flow_error *error)
5359 {
5360         struct mlx5_priv *priv = dev->data->dev_private;
5361         struct rte_flow_action *sfx_actions = NULL;
5362         struct rte_flow_action *pre_actions = NULL;
5363         struct rte_flow_item *sfx_items = NULL;
5364         struct mlx5_flow *dev_flow = NULL;
5365         struct rte_flow_attr sfx_attr = *attr;
5366 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5367         struct mlx5_flow_dv_sample_resource *sample_res;
5368         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5369         struct mlx5_flow_tbl_resource *sfx_tbl;
5370         union mlx5_flow_tbl_key sfx_table_key;
5371 #endif
5372         size_t act_size;
5373         size_t item_size;
5374         uint32_t fdb_tx = 0;
5375         int32_t tag_id = 0;
5376         int actions_n = 0;
5377         int sample_action_pos;
5378         int qrss_action_pos;
5379         int ret = 0;
5380
5381         if (priv->sampler_en)
5382                 actions_n = flow_check_match_action(actions, attr,
5383                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5384                                         &sample_action_pos, &qrss_action_pos);
5385         if (actions_n) {
5386                 /* The prefix actions must includes sample, tag, end. */
5387                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5388                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5389                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5390                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5391                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5392                                           item_size), 0, SOCKET_ID_ANY);
5393                 if (!sfx_actions)
5394                         return rte_flow_error_set(error, ENOMEM,
5395                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5396                                                   NULL, "no memory to split "
5397                                                   "sample flow");
5398                 /* The representor_id is -1 for uplink. */
5399                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5400                 if (!fdb_tx)
5401                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5402                                         + act_size);
5403                 pre_actions = sfx_actions + actions_n;
5404                 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items,
5405                                                 actions, sfx_actions,
5406                                                 pre_actions, actions_n,
5407                                                 sample_action_pos,
5408                                                 qrss_action_pos, error);
5409                 if (tag_id < 0 || (!fdb_tx && !tag_id)) {
5410                         ret = -rte_errno;
5411                         goto exit;
5412                 }
5413                 /* Add the prefix subflow. */
5414                 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr,
5415                                               items, pre_actions, external,
5416                                               flow_idx, error);
5417                 if (ret) {
5418                         ret = -rte_errno;
5419                         goto exit;
5420                 }
5421                 dev_flow->handle->split_flow_id = tag_id;
5422 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5423                 /* Set the sfx group attr. */
5424                 sample_res = (struct mlx5_flow_dv_sample_resource *)
5425                                         dev_flow->dv.sample_res;
5426                 sfx_tbl = (struct mlx5_flow_tbl_resource *)
5427                                         sample_res->normal_path_tbl;
5428                 sfx_tbl_data = container_of(sfx_tbl,
5429                                         struct mlx5_flow_tbl_data_entry, tbl);
5430                 sfx_table_key.v64 = sfx_tbl_data->entry.key;
5431                 sfx_attr.group = sfx_attr.transfer ?
5432                                         (sfx_table_key.table_id - 1) :
5433                                          sfx_table_key.table_id;
5434 #endif
5435         }
5436         /* Add the suffix subflow. */
5437         ret = flow_create_split_meter(dev, flow, dev_flow ?
5438                                  flow_get_prefix_layer_flags(dev_flow) : 0,
5439                                  dev_flow ? dev_flow->handle->mark : 0,
5440                                  &sfx_attr, sfx_items ? sfx_items : items,
5441                                  sfx_actions ? sfx_actions : actions,
5442                                  external, flow_idx, error);
5443 exit:
5444         if (sfx_actions)
5445                 mlx5_free(sfx_actions);
5446         return ret;
5447 }
5448
5449 /**
5450  * Split the flow to subflow set. The splitters might be linked
5451  * in the chain, like this:
5452  * flow_create_split_outer() calls:
5453  *   flow_create_split_meter() calls:
5454  *     flow_create_split_metadata(meter_subflow_0) calls:
5455  *       flow_create_split_inner(metadata_subflow_0)
5456  *       flow_create_split_inner(metadata_subflow_1)
5457  *       flow_create_split_inner(metadata_subflow_2)
5458  *     flow_create_split_metadata(meter_subflow_1) calls:
5459  *       flow_create_split_inner(metadata_subflow_0)
5460  *       flow_create_split_inner(metadata_subflow_1)
5461  *       flow_create_split_inner(metadata_subflow_2)
5462  *
5463  * This provide flexible way to add new levels of flow splitting.
5464  * The all of successfully created subflows are included to the
5465  * parent flow dev_flow list.
5466  *
5467  * @param dev
5468  *   Pointer to Ethernet device.
5469  * @param[in] flow
5470  *   Parent flow structure pointer.
5471  * @param[in] attr
5472  *   Flow rule attributes.
5473  * @param[in] items
5474  *   Pattern specification (list terminated by the END pattern item).
5475  * @param[in] actions
5476  *   Associated actions (list terminated by the END action).
5477  * @param[in] external
5478  *   This flow rule is created by request external to PMD.
5479  * @param[in] flow_idx
5480  *   This memory pool index to the flow.
5481  * @param[out] error
5482  *   Perform verbose error reporting if not NULL.
5483  * @return
5484  *   0 on success, negative value otherwise
5485  */
5486 static int
5487 flow_create_split_outer(struct rte_eth_dev *dev,
5488                         struct rte_flow *flow,
5489                         const struct rte_flow_attr *attr,
5490                         const struct rte_flow_item items[],
5491                         const struct rte_flow_action actions[],
5492                         bool external, uint32_t flow_idx,
5493                         struct rte_flow_error *error)
5494 {
5495         int ret;
5496
5497         ret = flow_create_split_sample(dev, flow, attr, items,
5498                                        actions, external, flow_idx, error);
5499         MLX5_ASSERT(ret <= 0);
5500         return ret;
5501 }
5502
5503 static struct mlx5_flow_tunnel *
5504 flow_tunnel_from_rule(struct rte_eth_dev *dev,
5505                       const struct rte_flow_attr *attr,
5506                       const struct rte_flow_item items[],
5507                       const struct rte_flow_action actions[])
5508 {
5509         struct mlx5_flow_tunnel *tunnel;
5510
5511 #pragma GCC diagnostic push
5512 #pragma GCC diagnostic ignored "-Wcast-qual"
5513         if (is_flow_tunnel_match_rule(dev, attr, items, actions))
5514                 tunnel = (struct mlx5_flow_tunnel *)items[0].spec;
5515         else if (is_flow_tunnel_steer_rule(dev, attr, items, actions))
5516                 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf;
5517         else
5518                 tunnel = NULL;
5519 #pragma GCC diagnostic pop
5520
5521         return tunnel;
5522 }
5523
5524 /**
5525  * Create a flow and add it to @p list.
5526  *
5527  * @param dev
5528  *   Pointer to Ethernet device.
5529  * @param list
5530  *   Pointer to a TAILQ flow list. If this parameter NULL,
5531  *   no list insertion occurred, flow is just created,
5532  *   this is caller's responsibility to track the
5533  *   created flow.
5534  * @param[in] attr
5535  *   Flow rule attributes.
5536  * @param[in] items
5537  *   Pattern specification (list terminated by the END pattern item).
5538  * @param[in] actions
5539  *   Associated actions (list terminated by the END action).
5540  * @param[in] external
5541  *   This flow rule is created by request external to PMD.
5542  * @param[out] error
5543  *   Perform verbose error reporting if not NULL.
5544  *
5545  * @return
5546  *   A flow index on success, 0 otherwise and rte_errno is set.
5547  */
5548 static uint32_t
5549 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
5550                  const struct rte_flow_attr *attr,
5551                  const struct rte_flow_item items[],
5552                  const struct rte_flow_action original_actions[],
5553                  bool external, struct rte_flow_error *error)
5554 {
5555         struct mlx5_priv *priv = dev->data->dev_private;
5556         struct rte_flow *flow = NULL;
5557         struct mlx5_flow *dev_flow;
5558         const struct rte_flow_action_rss *rss;
5559         struct mlx5_translated_shared_action
5560                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5561         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5562         union {
5563                 struct mlx5_flow_expand_rss buf;
5564                 uint8_t buffer[2048];
5565         } expand_buffer;
5566         union {
5567                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5568                 uint8_t buffer[2048];
5569         } actions_rx;
5570         union {
5571                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
5572                 uint8_t buffer[2048];
5573         } actions_hairpin_tx;
5574         union {
5575                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
5576                 uint8_t buffer[2048];
5577         } items_tx;
5578         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
5579         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
5580                                               priv->rss_desc)[!!priv->flow_idx];
5581         const struct rte_flow_action *p_actions_rx;
5582         uint32_t i;
5583         uint32_t idx = 0;
5584         int hairpin_flow;
5585         uint32_t hairpin_id = 0;
5586         struct rte_flow_attr attr_tx = { .priority = 0 };
5587         struct rte_flow_attr attr_factor = {0};
5588         const struct rte_flow_action *actions;
5589         struct rte_flow_action *translated_actions = NULL;
5590         struct mlx5_flow_tunnel *tunnel;
5591         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
5592         int ret = flow_shared_actions_translate(original_actions,
5593                                                 shared_actions,
5594                                                 &shared_actions_n,
5595                                                 &translated_actions, error);
5596
5597         if (ret < 0) {
5598                 MLX5_ASSERT(translated_actions == NULL);
5599                 return 0;
5600         }
5601         actions = translated_actions ? translated_actions : original_actions;
5602         memcpy((void *)&attr_factor, (const void *)attr, sizeof(*attr));
5603         p_actions_rx = actions;
5604         hairpin_flow = flow_check_hairpin_split(dev, &attr_factor, actions);
5605         ret = flow_drv_validate(dev, &attr_factor, items, p_actions_rx,
5606                                 external, hairpin_flow, error);
5607         if (ret < 0)
5608                 goto error_before_hairpin_split;
5609         if (hairpin_flow > 0) {
5610                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
5611                         rte_errno = EINVAL;
5612                         goto error_before_hairpin_split;
5613                 }
5614                 flow_hairpin_split(dev, actions, actions_rx.actions,
5615                                    actions_hairpin_tx.actions, items_tx.items,
5616                                    &hairpin_id);
5617                 p_actions_rx = actions_rx.actions;
5618         }
5619         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
5620         if (!flow) {
5621                 rte_errno = ENOMEM;
5622                 goto error_before_flow;
5623         }
5624         flow->drv_type = flow_get_drv_type(dev, &attr_factor);
5625         if (hairpin_id != 0)
5626                 flow->hairpin_flow_id = hairpin_id;
5627         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
5628                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
5629         memset(rss_desc, 0, sizeof(*rss_desc));
5630         rss = flow_get_rss_action(p_actions_rx);
5631         if (rss) {
5632                 /*
5633                  * The following information is required by
5634                  * mlx5_flow_hashfields_adjust() in advance.
5635                  */
5636                 rss_desc->level = rss->level;
5637                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
5638                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
5639         }
5640         flow->dev_handles = 0;
5641         if (rss && rss->types) {
5642                 unsigned int graph_root;
5643
5644                 graph_root = find_graph_root(items, rss->level);
5645                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
5646                                            items, rss->types,
5647                                            mlx5_support_expansion, graph_root);
5648                 MLX5_ASSERT(ret > 0 &&
5649                        (unsigned int)ret < sizeof(expand_buffer.buffer));
5650         } else {
5651                 buf->entries = 1;
5652                 buf->entry[0].pattern = (void *)(uintptr_t)items;
5653         }
5654         flow->shared_rss = flow_get_shared_rss_action(shared_actions,
5655                                                       shared_actions_n);
5656         /*
5657          * Record the start index when there is a nested call. All sub-flows
5658          * need to be translated before another calling.
5659          * No need to use ping-pong buffer to save memory here.
5660          */
5661         if (priv->flow_idx) {
5662                 MLX5_ASSERT(!priv->flow_nested_idx);
5663                 priv->flow_nested_idx = priv->flow_idx;
5664         }
5665         for (i = 0; i < buf->entries; ++i) {
5666                 /*
5667                  * The splitter may create multiple dev_flows,
5668                  * depending on configuration. In the simplest
5669                  * case it just creates unmodified original flow.
5670                  */
5671                 ret = flow_create_split_outer(dev, flow, &attr_factor,
5672                                               buf->entry[i].pattern,
5673                                               p_actions_rx, external, idx,
5674                                               error);
5675                 if (ret < 0)
5676                         goto error;
5677                 if (is_flow_tunnel_steer_rule(dev, attr,
5678                                               buf->entry[i].pattern,
5679                                               p_actions_rx)) {
5680                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
5681                                                            p_actions_rx,
5682                                                            idx,
5683                                                            &default_miss_ctx,
5684                                                            error);
5685                         if (ret < 0) {
5686                                 mlx5_free(default_miss_ctx.queue);
5687                                 goto error;
5688                         }
5689                 }
5690         }
5691         /* Create the tx flow. */
5692         if (hairpin_flow) {
5693                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
5694                 attr_tx.ingress = 0;
5695                 attr_tx.egress = 1;
5696                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
5697                                          actions_hairpin_tx.actions,
5698                                          idx, error);
5699                 if (!dev_flow)
5700                         goto error;
5701                 dev_flow->flow = flow;
5702                 dev_flow->external = 0;
5703                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5704                               dev_flow->handle, next);
5705                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
5706                                          items_tx.items,
5707                                          actions_hairpin_tx.actions, error);
5708                 if (ret < 0)
5709                         goto error;
5710         }
5711         /*
5712          * Update the metadata register copy table. If extensive
5713          * metadata feature is enabled and registers are supported
5714          * we might create the extra rte_flow for each unique
5715          * MARK/FLAG action ID.
5716          *
5717          * The table is updated for ingress Flows only, because
5718          * the egress Flows belong to the different device and
5719          * copy table should be updated in peer NIC Rx domain.
5720          */
5721         if (attr_factor.ingress &&
5722             (external || attr_factor.group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
5723                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
5724                 if (ret)
5725                         goto error;
5726         }
5727         /*
5728          * If the flow is external (from application) OR device is started, then
5729          * the flow will be applied immediately.
5730          */
5731         if (external || dev->data->dev_started) {
5732                 ret = flow_drv_apply(dev, flow, error);
5733                 if (ret < 0)
5734                         goto error;
5735         }
5736         if (list)
5737                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
5738                              flow, next);
5739         flow_rxq_flags_set(dev, flow);
5740         rte_free(translated_actions);
5741         /* Nested flow creation index recovery. */
5742         priv->flow_idx = priv->flow_nested_idx;
5743         if (priv->flow_nested_idx)
5744                 priv->flow_nested_idx = 0;
5745         tunnel = flow_tunnel_from_rule(dev, attr, items, actions);
5746         if (tunnel) {
5747                 flow->tunnel = 1;
5748                 flow->tunnel_id = tunnel->tunnel_id;
5749                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
5750                 mlx5_free(default_miss_ctx.queue);
5751         }
5752         return idx;
5753 error:
5754         MLX5_ASSERT(flow);
5755         ret = rte_errno; /* Save rte_errno before cleanup. */
5756         flow_mreg_del_copy_action(dev, flow);
5757         flow_drv_destroy(dev, flow);
5758         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
5759         rte_errno = ret; /* Restore rte_errno. */
5760 error_before_flow:
5761         ret = rte_errno;
5762         if (hairpin_id)
5763                 mlx5_flow_id_release(priv->sh->flow_id_pool,
5764                                      hairpin_id);
5765         rte_errno = ret;
5766         priv->flow_idx = priv->flow_nested_idx;
5767         if (priv->flow_nested_idx)
5768                 priv->flow_nested_idx = 0;
5769 error_before_hairpin_split:
5770         rte_free(translated_actions);
5771         return 0;
5772 }
5773
5774 /**
5775  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
5776  * incoming packets to table 1.
5777  *
5778  * Other flow rules, requested for group n, will be created in
5779  * e-switch table n+1.
5780  * Jump action to e-switch group n will be created to group n+1.
5781  *
5782  * Used when working in switchdev mode, to utilise advantages of table 1
5783  * and above.
5784  *
5785  * @param dev
5786  *   Pointer to Ethernet device.
5787  *
5788  * @return
5789  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
5790  */
5791 struct rte_flow *
5792 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
5793 {
5794         const struct rte_flow_attr attr = {
5795                 .group = 0,
5796                 .priority = 0,
5797                 .ingress = 1,
5798                 .egress = 0,
5799                 .transfer = 1,
5800         };
5801         const struct rte_flow_item pattern = {
5802                 .type = RTE_FLOW_ITEM_TYPE_END,
5803         };
5804         struct rte_flow_action_jump jump = {
5805                 .group = 1,
5806         };
5807         const struct rte_flow_action actions[] = {
5808                 {
5809                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
5810                         .conf = &jump,
5811                 },
5812                 {
5813                         .type = RTE_FLOW_ACTION_TYPE_END,
5814                 },
5815         };
5816         struct mlx5_priv *priv = dev->data->dev_private;
5817         struct rte_flow_error error;
5818
5819         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
5820                                                    &attr, &pattern,
5821                                                    actions, false, &error);
5822 }
5823
5824 /**
5825  * Validate a flow supported by the NIC.
5826  *
5827  * @see rte_flow_validate()
5828  * @see rte_flow_ops
5829  */
5830 int
5831 mlx5_flow_validate(struct rte_eth_dev *dev,
5832                    const struct rte_flow_attr *attr,
5833                    const struct rte_flow_item items[],
5834                    const struct rte_flow_action original_actions[],
5835                    struct rte_flow_error *error)
5836 {
5837         int hairpin_flow;
5838         struct mlx5_translated_shared_action
5839                 shared_actions[MLX5_MAX_SHARED_ACTIONS];
5840         int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
5841         const struct rte_flow_action *actions;
5842         struct rte_flow_action *translated_actions = NULL;
5843         int ret = flow_shared_actions_translate(original_actions,
5844                                                 shared_actions,
5845                                                 &shared_actions_n,
5846                                                 &translated_actions, error);
5847
5848         if (ret)
5849                 return ret;
5850         actions = translated_actions ? translated_actions : original_actions;
5851         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
5852         ret = flow_drv_validate(dev, attr, items, actions,
5853                                 true, hairpin_flow, error);
5854         rte_free(translated_actions);
5855         return ret;
5856 }
5857
5858 /**
5859  * Create a flow.
5860  *
5861  * @see rte_flow_create()
5862  * @see rte_flow_ops
5863  */
5864 struct rte_flow *
5865 mlx5_flow_create(struct rte_eth_dev *dev,
5866                  const struct rte_flow_attr *attr,
5867                  const struct rte_flow_item items[],
5868                  const struct rte_flow_action actions[],
5869                  struct rte_flow_error *error)
5870 {
5871         struct mlx5_priv *priv = dev->data->dev_private;
5872
5873         /*
5874          * If the device is not started yet, it is not allowed to created a
5875          * flow from application. PMD default flows and traffic control flows
5876          * are not affected.
5877          */
5878         if (unlikely(!dev->data->dev_started)) {
5879                 DRV_LOG(DEBUG, "port %u is not started when "
5880                         "inserting a flow", dev->data->port_id);
5881                 rte_flow_error_set(error, ENODEV,
5882                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5883                                    NULL,
5884                                    "port not started");
5885                 return NULL;
5886         }
5887
5888         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
5889                                   attr, items, actions, true, error);
5890 }
5891
5892 /**
5893  * Destroy a flow in a list.
5894  *
5895  * @param dev
5896  *   Pointer to Ethernet device.
5897  * @param list
5898  *   Pointer to the Indexed flow list. If this parameter NULL,
5899  *   there is no flow removal from the list. Be noted that as
5900  *   flow is add to the indexed list, memory of the indexed
5901  *   list points to maybe changed as flow destroyed.
5902  * @param[in] flow_idx
5903  *   Index of flow to destroy.
5904  */
5905 static void
5906 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
5907                   uint32_t flow_idx)
5908 {
5909         struct mlx5_priv *priv = dev->data->dev_private;
5910         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5911         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5912                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
5913
5914         if (!flow)
5915                 return;
5916         /*
5917          * Update RX queue flags only if port is started, otherwise it is
5918          * already clean.
5919          */
5920         if (dev->data->dev_started)
5921                 flow_rxq_flags_trim(dev, flow);
5922         if (flow->hairpin_flow_id)
5923                 mlx5_flow_id_release(priv->sh->flow_id_pool,
5924                                      flow->hairpin_flow_id);
5925         flow_drv_destroy(dev, flow);
5926         if (list)
5927                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
5928                              flow_idx, flow, next);
5929         flow_mreg_del_copy_action(dev, flow);
5930         if (flow->fdir) {
5931                 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5932                         if (priv_fdir_flow->rix_flow == flow_idx)
5933                                 break;
5934                 }
5935                 if (priv_fdir_flow) {
5936                         LIST_REMOVE(priv_fdir_flow, next);
5937                         mlx5_free(priv_fdir_flow->fdir);
5938                         mlx5_free(priv_fdir_flow);
5939                 }
5940         }
5941         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5942         if (flow->tunnel) {
5943                 struct mlx5_flow_tunnel *tunnel;
5944                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
5945                 RTE_VERIFY(tunnel);
5946                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
5947                         mlx5_flow_tunnel_free(dev, tunnel);
5948         }
5949 }
5950
5951 /**
5952  * Destroy all flows.
5953  *
5954  * @param dev
5955  *   Pointer to Ethernet device.
5956  * @param list
5957  *   Pointer to the Indexed flow list.
5958  * @param active
5959  *   If flushing is called avtively.
5960  */
5961 void
5962 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
5963 {
5964         uint32_t num_flushed = 0;
5965
5966         while (*list) {
5967                 flow_list_destroy(dev, list, *list);
5968                 num_flushed++;
5969         }
5970         if (active) {
5971                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5972                         dev->data->port_id, num_flushed);
5973         }
5974 }
5975
5976 /**
5977  * Remove all flows.
5978  *
5979  * @param dev
5980  *   Pointer to Ethernet device.
5981  * @param list
5982  *   Pointer to the Indexed flow list.
5983  */
5984 void
5985 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
5986 {
5987         struct mlx5_priv *priv = dev->data->dev_private;
5988         struct rte_flow *flow = NULL;
5989         uint32_t idx;
5990
5991         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
5992                       flow, next) {
5993                 flow_drv_remove(dev, flow);
5994                 flow_mreg_stop_copy_action(dev, flow);
5995         }
5996         flow_mreg_del_default_copy_action(dev);
5997         flow_rxq_flags_clear(dev);
5998 }
5999
6000 /**
6001  * Add all flows.
6002  *
6003  * @param dev
6004  *   Pointer to Ethernet device.
6005  * @param list
6006  *   Pointer to the Indexed flow list.
6007  *
6008  * @return
6009  *   0 on success, a negative errno value otherwise and rte_errno is set.
6010  */
6011 int
6012 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
6013 {
6014         struct mlx5_priv *priv = dev->data->dev_private;
6015         struct rte_flow *flow = NULL;
6016         struct rte_flow_error error;
6017         uint32_t idx;
6018         int ret = 0;
6019
6020         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6021         ret = flow_mreg_add_default_copy_action(dev, &error);
6022         if (ret < 0)
6023                 return -rte_errno;
6024         /* Apply Flows created by application. */
6025         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
6026                       flow, next) {
6027                 ret = flow_mreg_start_copy_action(dev, flow);
6028                 if (ret < 0)
6029                         goto error;
6030                 ret = flow_drv_apply(dev, flow, &error);
6031                 if (ret < 0)
6032                         goto error;
6033                 flow_rxq_flags_set(dev, flow);
6034         }
6035         return 0;
6036 error:
6037         ret = rte_errno; /* Save rte_errno before cleanup. */
6038         mlx5_flow_stop(dev, list);
6039         rte_errno = ret; /* Restore rte_errno. */
6040         return -rte_errno;
6041 }
6042
6043 /**
6044  * Stop all default actions for flows.
6045  *
6046  * @param dev
6047  *   Pointer to Ethernet device.
6048  */
6049 void
6050 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6051 {
6052         flow_mreg_del_default_copy_action(dev);
6053         flow_rxq_flags_clear(dev);
6054 }
6055
6056 /**
6057  * Start all default actions for flows.
6058  *
6059  * @param dev
6060  *   Pointer to Ethernet device.
6061  * @return
6062  *   0 on success, a negative errno value otherwise and rte_errno is set.
6063  */
6064 int
6065 mlx5_flow_start_default(struct rte_eth_dev *dev)
6066 {
6067         struct rte_flow_error error;
6068
6069         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6070         return flow_mreg_add_default_copy_action(dev, &error);
6071 }
6072
6073 /**
6074  * Allocate intermediate resources for flow creation.
6075  *
6076  * @param dev
6077  *   Pointer to Ethernet device.
6078  */
6079 void
6080 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
6081 {
6082         struct mlx5_priv *priv = dev->data->dev_private;
6083
6084         if (!priv->inter_flows) {
6085                 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO,
6086                                     MLX5_NUM_MAX_DEV_FLOWS *
6087                                     sizeof(struct mlx5_flow) +
6088                                     (sizeof(struct mlx5_flow_rss_desc) +
6089                                     sizeof(uint16_t) * UINT16_MAX) * 2, 0,
6090                                     SOCKET_ID_ANY);
6091                 if (!priv->inter_flows) {
6092                         DRV_LOG(ERR, "can't allocate intermediate memory.");
6093                         return;
6094                 }
6095         }
6096         priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
6097                          [MLX5_NUM_MAX_DEV_FLOWS];
6098         /* Reset the index. */
6099         priv->flow_idx = 0;
6100         priv->flow_nested_idx = 0;
6101 }
6102
6103 /**
6104  * Free intermediate resources for flows.
6105  *
6106  * @param dev
6107  *   Pointer to Ethernet device.
6108  */
6109 void
6110 mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
6111 {
6112         struct mlx5_priv *priv = dev->data->dev_private;
6113
6114         mlx5_free(priv->inter_flows);
6115         priv->inter_flows = NULL;
6116 }
6117
6118 /**
6119  * Verify the flow list is empty
6120  *
6121  * @param dev
6122  *  Pointer to Ethernet device.
6123  *
6124  * @return the number of flows not released.
6125  */
6126 int
6127 mlx5_flow_verify(struct rte_eth_dev *dev)
6128 {
6129         struct mlx5_priv *priv = dev->data->dev_private;
6130         struct rte_flow *flow;
6131         uint32_t idx;
6132         int ret = 0;
6133
6134         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6135                       flow, next) {
6136                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6137                         dev->data->port_id, (void *)flow);
6138                 ++ret;
6139         }
6140         return ret;
6141 }
6142
6143 /**
6144  * Enable default hairpin egress flow.
6145  *
6146  * @param dev
6147  *   Pointer to Ethernet device.
6148  * @param queue
6149  *   The queue index.
6150  *
6151  * @return
6152  *   0 on success, a negative errno value otherwise and rte_errno is set.
6153  */
6154 int
6155 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6156                             uint32_t queue)
6157 {
6158         struct mlx5_priv *priv = dev->data->dev_private;
6159         const struct rte_flow_attr attr = {
6160                 .egress = 1,
6161                 .priority = 0,
6162         };
6163         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6164                 .queue = queue,
6165         };
6166         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6167                 .queue = UINT32_MAX,
6168         };
6169         struct rte_flow_item items[] = {
6170                 {
6171                         .type = (enum rte_flow_item_type)
6172                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6173                         .spec = &queue_spec,
6174                         .last = NULL,
6175                         .mask = &queue_mask,
6176                 },
6177                 {
6178                         .type = RTE_FLOW_ITEM_TYPE_END,
6179                 },
6180         };
6181         struct rte_flow_action_jump jump = {
6182                 .group = MLX5_HAIRPIN_TX_TABLE,
6183         };
6184         struct rte_flow_action actions[2];
6185         uint32_t flow_idx;
6186         struct rte_flow_error error;
6187
6188         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6189         actions[0].conf = &jump;
6190         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6191         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6192                                 &attr, items, actions, false, &error);
6193         if (!flow_idx) {
6194                 DRV_LOG(DEBUG,
6195                         "Failed to create ctrl flow: rte_errno(%d),"
6196                         " type(%d), message(%s)",
6197                         rte_errno, error.type,
6198                         error.message ? error.message : " (no stated reason)");
6199                 return -rte_errno;
6200         }
6201         return 0;
6202 }
6203
6204 /**
6205  * Enable a control flow configured from the control plane.
6206  *
6207  * @param dev
6208  *   Pointer to Ethernet device.
6209  * @param eth_spec
6210  *   An Ethernet flow spec to apply.
6211  * @param eth_mask
6212  *   An Ethernet flow mask to apply.
6213  * @param vlan_spec
6214  *   A VLAN flow spec to apply.
6215  * @param vlan_mask
6216  *   A VLAN flow mask to apply.
6217  *
6218  * @return
6219  *   0 on success, a negative errno value otherwise and rte_errno is set.
6220  */
6221 int
6222 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6223                     struct rte_flow_item_eth *eth_spec,
6224                     struct rte_flow_item_eth *eth_mask,
6225                     struct rte_flow_item_vlan *vlan_spec,
6226                     struct rte_flow_item_vlan *vlan_mask)
6227 {
6228         struct mlx5_priv *priv = dev->data->dev_private;
6229         const struct rte_flow_attr attr = {
6230                 .ingress = 1,
6231                 .priority = MLX5_FLOW_PRIO_RSVD,
6232         };
6233         struct rte_flow_item items[] = {
6234                 {
6235                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6236                         .spec = eth_spec,
6237                         .last = NULL,
6238                         .mask = eth_mask,
6239                 },
6240                 {
6241                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6242                                               RTE_FLOW_ITEM_TYPE_END,
6243                         .spec = vlan_spec,
6244                         .last = NULL,
6245                         .mask = vlan_mask,
6246                 },
6247                 {
6248                         .type = RTE_FLOW_ITEM_TYPE_END,
6249                 },
6250         };
6251         uint16_t queue[priv->reta_idx_n];
6252         struct rte_flow_action_rss action_rss = {
6253                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6254                 .level = 0,
6255                 .types = priv->rss_conf.rss_hf,
6256                 .key_len = priv->rss_conf.rss_key_len,
6257                 .queue_num = priv->reta_idx_n,
6258                 .key = priv->rss_conf.rss_key,
6259                 .queue = queue,
6260         };
6261         struct rte_flow_action actions[] = {
6262                 {
6263                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6264                         .conf = &action_rss,
6265                 },
6266                 {
6267                         .type = RTE_FLOW_ACTION_TYPE_END,
6268                 },
6269         };
6270         uint32_t flow_idx;
6271         struct rte_flow_error error;
6272         unsigned int i;
6273
6274         if (!priv->reta_idx_n || !priv->rxqs_n) {
6275                 return 0;
6276         }
6277         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6278                 action_rss.types = 0;
6279         for (i = 0; i != priv->reta_idx_n; ++i)
6280                 queue[i] = (*priv->reta_idx)[i];
6281         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6282                                 &attr, items, actions, false, &error);
6283         if (!flow_idx)
6284                 return -rte_errno;
6285         return 0;
6286 }
6287
6288 /**
6289  * Enable a flow control configured from the control plane.
6290  *
6291  * @param dev
6292  *   Pointer to Ethernet device.
6293  * @param eth_spec
6294  *   An Ethernet flow spec to apply.
6295  * @param eth_mask
6296  *   An Ethernet flow mask to apply.
6297  *
6298  * @return
6299  *   0 on success, a negative errno value otherwise and rte_errno is set.
6300  */
6301 int
6302 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6303                struct rte_flow_item_eth *eth_spec,
6304                struct rte_flow_item_eth *eth_mask)
6305 {
6306         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6307 }
6308
6309 /**
6310  * Create default miss flow rule matching lacp traffic
6311  *
6312  * @param dev
6313  *   Pointer to Ethernet device.
6314  * @param eth_spec
6315  *   An Ethernet flow spec to apply.
6316  *
6317  * @return
6318  *   0 on success, a negative errno value otherwise and rte_errno is set.
6319  */
6320 int
6321 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6322 {
6323         struct mlx5_priv *priv = dev->data->dev_private;
6324         /*
6325          * The LACP matching is done by only using ether type since using
6326          * a multicast dst mac causes kernel to give low priority to this flow.
6327          */
6328         static const struct rte_flow_item_eth lacp_spec = {
6329                 .type = RTE_BE16(0x8809),
6330         };
6331         static const struct rte_flow_item_eth lacp_mask = {
6332                 .type = 0xffff,
6333         };
6334         const struct rte_flow_attr attr = {
6335                 .ingress = 1,
6336         };
6337         struct rte_flow_item items[] = {
6338                 {
6339                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6340                         .spec = &lacp_spec,
6341                         .mask = &lacp_mask,
6342                 },
6343                 {
6344                         .type = RTE_FLOW_ITEM_TYPE_END,
6345                 },
6346         };
6347         struct rte_flow_action actions[] = {
6348                 {
6349                         .type = (enum rte_flow_action_type)
6350                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6351                 },
6352                 {
6353                         .type = RTE_FLOW_ACTION_TYPE_END,
6354                 },
6355         };
6356         struct rte_flow_error error;
6357         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6358                                 &attr, items, actions, false, &error);
6359
6360         if (!flow_idx)
6361                 return -rte_errno;
6362         return 0;
6363 }
6364
6365 /**
6366  * Destroy a flow.
6367  *
6368  * @see rte_flow_destroy()
6369  * @see rte_flow_ops
6370  */
6371 int
6372 mlx5_flow_destroy(struct rte_eth_dev *dev,
6373                   struct rte_flow *flow,
6374                   struct rte_flow_error *error __rte_unused)
6375 {
6376         struct mlx5_priv *priv = dev->data->dev_private;
6377
6378         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6379         return 0;
6380 }
6381
6382 /**
6383  * Destroy all flows.
6384  *
6385  * @see rte_flow_flush()
6386  * @see rte_flow_ops
6387  */
6388 int
6389 mlx5_flow_flush(struct rte_eth_dev *dev,
6390                 struct rte_flow_error *error __rte_unused)
6391 {
6392         struct mlx5_priv *priv = dev->data->dev_private;
6393
6394         mlx5_flow_list_flush(dev, &priv->flows, false);
6395         return 0;
6396 }
6397
6398 /**
6399  * Isolated mode.
6400  *
6401  * @see rte_flow_isolate()
6402  * @see rte_flow_ops
6403  */
6404 int
6405 mlx5_flow_isolate(struct rte_eth_dev *dev,
6406                   int enable,
6407                   struct rte_flow_error *error)
6408 {
6409         struct mlx5_priv *priv = dev->data->dev_private;
6410
6411         if (dev->data->dev_started) {
6412                 rte_flow_error_set(error, EBUSY,
6413                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6414                                    NULL,
6415                                    "port must be stopped first");
6416                 return -rte_errno;
6417         }
6418         priv->isolated = !!enable;
6419         if (enable)
6420                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
6421         else
6422                 dev->dev_ops = &mlx5_os_dev_ops;
6423
6424         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6425         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6426
6427         return 0;
6428 }
6429
6430 /**
6431  * Query a flow.
6432  *
6433  * @see rte_flow_query()
6434  * @see rte_flow_ops
6435  */
6436 static int
6437 flow_drv_query(struct rte_eth_dev *dev,
6438                uint32_t flow_idx,
6439                const struct rte_flow_action *actions,
6440                void *data,
6441                struct rte_flow_error *error)
6442 {
6443         struct mlx5_priv *priv = dev->data->dev_private;
6444         const struct mlx5_flow_driver_ops *fops;
6445         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6446                                                [MLX5_IPOOL_RTE_FLOW],
6447                                                flow_idx);
6448         enum mlx5_flow_drv_type ftype;
6449
6450         if (!flow) {
6451                 return rte_flow_error_set(error, ENOENT,
6452                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6453                           NULL,
6454                           "invalid flow handle");
6455         }
6456         ftype = flow->drv_type;
6457         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
6458         fops = flow_get_drv_ops(ftype);
6459
6460         return fops->query(dev, flow, actions, data, error);
6461 }
6462
6463 /**
6464  * Query a flow.
6465  *
6466  * @see rte_flow_query()
6467  * @see rte_flow_ops
6468  */
6469 int
6470 mlx5_flow_query(struct rte_eth_dev *dev,
6471                 struct rte_flow *flow,
6472                 const struct rte_flow_action *actions,
6473                 void *data,
6474                 struct rte_flow_error *error)
6475 {
6476         int ret;
6477
6478         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
6479                              error);
6480         if (ret < 0)
6481                 return ret;
6482         return 0;
6483 }
6484
6485 /**
6486  * Convert a flow director filter to a generic flow.
6487  *
6488  * @param dev
6489  *   Pointer to Ethernet device.
6490  * @param fdir_filter
6491  *   Flow director filter to add.
6492  * @param attributes
6493  *   Generic flow parameters structure.
6494  *
6495  * @return
6496  *   0 on success, a negative errno value otherwise and rte_errno is set.
6497  */
6498 static int
6499 flow_fdir_filter_convert(struct rte_eth_dev *dev,
6500                          const struct rte_eth_fdir_filter *fdir_filter,
6501                          struct mlx5_fdir *attributes)
6502 {
6503         struct mlx5_priv *priv = dev->data->dev_private;
6504         const struct rte_eth_fdir_input *input = &fdir_filter->input;
6505         const struct rte_eth_fdir_masks *mask =
6506                 &dev->data->dev_conf.fdir_conf.mask;
6507
6508         /* Validate queue number. */
6509         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
6510                 DRV_LOG(ERR, "port %u invalid queue number %d",
6511                         dev->data->port_id, fdir_filter->action.rx_queue);
6512                 rte_errno = EINVAL;
6513                 return -rte_errno;
6514         }
6515         attributes->attr.ingress = 1;
6516         attributes->items[0] = (struct rte_flow_item) {
6517                 .type = RTE_FLOW_ITEM_TYPE_ETH,
6518                 .spec = &attributes->l2,
6519                 .mask = &attributes->l2_mask,
6520         };
6521         switch (fdir_filter->action.behavior) {
6522         case RTE_ETH_FDIR_ACCEPT:
6523                 attributes->actions[0] = (struct rte_flow_action){
6524                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
6525                         .conf = &attributes->queue,
6526                 };
6527                 break;
6528         case RTE_ETH_FDIR_REJECT:
6529                 attributes->actions[0] = (struct rte_flow_action){
6530                         .type = RTE_FLOW_ACTION_TYPE_DROP,
6531                 };
6532                 break;
6533         default:
6534                 DRV_LOG(ERR, "port %u invalid behavior %d",
6535                         dev->data->port_id,
6536                         fdir_filter->action.behavior);
6537                 rte_errno = ENOTSUP;
6538                 return -rte_errno;
6539         }
6540         attributes->queue.index = fdir_filter->action.rx_queue;
6541         /* Handle L3. */
6542         switch (fdir_filter->input.flow_type) {
6543         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
6544         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
6545         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
6546                 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
6547                         .src_addr = input->flow.ip4_flow.src_ip,
6548                         .dst_addr = input->flow.ip4_flow.dst_ip,
6549                         .time_to_live = input->flow.ip4_flow.ttl,
6550                         .type_of_service = input->flow.ip4_flow.tos,
6551                 };
6552                 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
6553                         .src_addr = mask->ipv4_mask.src_ip,
6554                         .dst_addr = mask->ipv4_mask.dst_ip,
6555                         .time_to_live = mask->ipv4_mask.ttl,
6556                         .type_of_service = mask->ipv4_mask.tos,
6557                         .next_proto_id = mask->ipv4_mask.proto,
6558                 };
6559                 attributes->items[1] = (struct rte_flow_item){
6560                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
6561                         .spec = &attributes->l3,
6562                         .mask = &attributes->l3_mask,
6563                 };
6564                 break;
6565         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
6566         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
6567         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
6568                 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
6569                         .hop_limits = input->flow.ipv6_flow.hop_limits,
6570                         .proto = input->flow.ipv6_flow.proto,
6571                 };
6572
6573                 memcpy(attributes->l3.ipv6.hdr.src_addr,
6574                        input->flow.ipv6_flow.src_ip,
6575                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
6576                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
6577                        input->flow.ipv6_flow.dst_ip,
6578                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
6579                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
6580                        mask->ipv6_mask.src_ip,
6581                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
6582                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
6583                        mask->ipv6_mask.dst_ip,
6584                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
6585                 attributes->items[1] = (struct rte_flow_item){
6586                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
6587                         .spec = &attributes->l3,
6588                         .mask = &attributes->l3_mask,
6589                 };
6590                 break;
6591         default:
6592                 DRV_LOG(ERR, "port %u invalid flow type%d",
6593                         dev->data->port_id, fdir_filter->input.flow_type);
6594                 rte_errno = ENOTSUP;
6595                 return -rte_errno;
6596         }
6597         /* Handle L4. */
6598         switch (fdir_filter->input.flow_type) {
6599         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
6600                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
6601                         .src_port = input->flow.udp4_flow.src_port,
6602                         .dst_port = input->flow.udp4_flow.dst_port,
6603                 };
6604                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
6605                         .src_port = mask->src_port_mask,
6606                         .dst_port = mask->dst_port_mask,
6607                 };
6608                 attributes->items[2] = (struct rte_flow_item){
6609                         .type = RTE_FLOW_ITEM_TYPE_UDP,
6610                         .spec = &attributes->l4,
6611                         .mask = &attributes->l4_mask,
6612                 };
6613                 break;
6614         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
6615                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
6616                         .src_port = input->flow.tcp4_flow.src_port,
6617                         .dst_port = input->flow.tcp4_flow.dst_port,
6618                 };
6619                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
6620                         .src_port = mask->src_port_mask,
6621                         .dst_port = mask->dst_port_mask,
6622                 };
6623                 attributes->items[2] = (struct rte_flow_item){
6624                         .type = RTE_FLOW_ITEM_TYPE_TCP,
6625                         .spec = &attributes->l4,
6626                         .mask = &attributes->l4_mask,
6627                 };
6628                 break;
6629         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
6630                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
6631                         .src_port = input->flow.udp6_flow.src_port,
6632                         .dst_port = input->flow.udp6_flow.dst_port,
6633                 };
6634                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
6635                         .src_port = mask->src_port_mask,
6636                         .dst_port = mask->dst_port_mask,
6637                 };
6638                 attributes->items[2] = (struct rte_flow_item){
6639                         .type = RTE_FLOW_ITEM_TYPE_UDP,
6640                         .spec = &attributes->l4,
6641                         .mask = &attributes->l4_mask,
6642                 };
6643                 break;
6644         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
6645                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
6646                         .src_port = input->flow.tcp6_flow.src_port,
6647                         .dst_port = input->flow.tcp6_flow.dst_port,
6648                 };
6649                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
6650                         .src_port = mask->src_port_mask,
6651                         .dst_port = mask->dst_port_mask,
6652                 };
6653                 attributes->items[2] = (struct rte_flow_item){
6654                         .type = RTE_FLOW_ITEM_TYPE_TCP,
6655                         .spec = &attributes->l4,
6656                         .mask = &attributes->l4_mask,
6657                 };
6658                 break;
6659         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
6660         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
6661                 break;
6662         default:
6663                 DRV_LOG(ERR, "port %u invalid flow type%d",
6664                         dev->data->port_id, fdir_filter->input.flow_type);
6665                 rte_errno = ENOTSUP;
6666                 return -rte_errno;
6667         }
6668         return 0;
6669 }
6670
6671 #define FLOW_FDIR_CMP(f1, f2, fld) \
6672         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
6673
6674 /**
6675  * Compare two FDIR flows. If items and actions are identical, the two flows are
6676  * regarded as same.
6677  *
6678  * @param dev
6679  *   Pointer to Ethernet device.
6680  * @param f1
6681  *   FDIR flow to compare.
6682  * @param f2
6683  *   FDIR flow to compare.
6684  *
6685  * @return
6686  *   Zero on match, 1 otherwise.
6687  */
6688 static int
6689 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
6690 {
6691         if (FLOW_FDIR_CMP(f1, f2, attr) ||
6692             FLOW_FDIR_CMP(f1, f2, l2) ||
6693             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
6694             FLOW_FDIR_CMP(f1, f2, l3) ||
6695             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
6696             FLOW_FDIR_CMP(f1, f2, l4) ||
6697             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
6698             FLOW_FDIR_CMP(f1, f2, actions[0].type))
6699                 return 1;
6700         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
6701             FLOW_FDIR_CMP(f1, f2, queue))
6702                 return 1;
6703         return 0;
6704 }
6705
6706 /**
6707  * Search device flow list to find out a matched FDIR flow.
6708  *
6709  * @param dev
6710  *   Pointer to Ethernet device.
6711  * @param fdir_flow
6712  *   FDIR flow to lookup.
6713  *
6714  * @return
6715  *   Index of flow if found, 0 otherwise.
6716  */
6717 static uint32_t
6718 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
6719 {
6720         struct mlx5_priv *priv = dev->data->dev_private;
6721         uint32_t flow_idx = 0;
6722         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6723
6724         MLX5_ASSERT(fdir_flow);
6725         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
6726                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
6727                         DRV_LOG(DEBUG, "port %u found FDIR flow %u",
6728                                 dev->data->port_id, flow_idx);
6729                         flow_idx = priv_fdir_flow->rix_flow;
6730                         break;
6731                 }
6732         }
6733         return flow_idx;
6734 }
6735
6736 /**
6737  * Add new flow director filter and store it in list.
6738  *
6739  * @param dev
6740  *   Pointer to Ethernet device.
6741  * @param fdir_filter
6742  *   Flow director filter to add.
6743  *
6744  * @return
6745  *   0 on success, a negative errno value otherwise and rte_errno is set.
6746  */
6747 static int
6748 flow_fdir_filter_add(struct rte_eth_dev *dev,
6749                      const struct rte_eth_fdir_filter *fdir_filter)
6750 {
6751         struct mlx5_priv *priv = dev->data->dev_private;
6752         struct mlx5_fdir *fdir_flow;
6753         struct rte_flow *flow;
6754         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6755         uint32_t flow_idx;
6756         int ret;
6757
6758         fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0,
6759                                 SOCKET_ID_ANY);
6760         if (!fdir_flow) {
6761                 rte_errno = ENOMEM;
6762                 return -rte_errno;
6763         }
6764         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
6765         if (ret)
6766                 goto error;
6767         flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
6768         if (flow_idx) {
6769                 rte_errno = EEXIST;
6770                 goto error;
6771         }
6772         priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO,
6773                                      sizeof(struct mlx5_fdir_flow),
6774                                      0, SOCKET_ID_ANY);
6775         if (!priv_fdir_flow) {
6776                 rte_errno = ENOMEM;
6777                 goto error;
6778         }
6779         flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
6780                                     fdir_flow->items, fdir_flow->actions, true,
6781                                     NULL);
6782         flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6783         if (!flow)
6784                 goto error;
6785         flow->fdir = 1;
6786         priv_fdir_flow->fdir = fdir_flow;
6787         priv_fdir_flow->rix_flow = flow_idx;
6788         LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
6789         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
6790                 dev->data->port_id, (void *)flow);
6791         return 0;
6792 error:
6793         mlx5_free(priv_fdir_flow);
6794         mlx5_free(fdir_flow);
6795         return -rte_errno;
6796 }
6797
6798 /**
6799  * Delete specific filter.
6800  *
6801  * @param dev
6802  *   Pointer to Ethernet device.
6803  * @param fdir_filter
6804  *   Filter to be deleted.
6805  *
6806  * @return
6807  *   0 on success, a negative errno value otherwise and rte_errno is set.
6808  */
6809 static int
6810 flow_fdir_filter_delete(struct rte_eth_dev *dev,
6811                         const struct rte_eth_fdir_filter *fdir_filter)
6812 {
6813         struct mlx5_priv *priv = dev->data->dev_private;
6814         uint32_t flow_idx;
6815         struct mlx5_fdir fdir_flow = {
6816                 .attr.group = 0,
6817         };
6818         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6819         int ret;
6820
6821         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
6822         if (ret)
6823                 return -rte_errno;
6824         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
6825                 /* Find the fdir in priv list */
6826                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
6827                         break;
6828         }
6829         if (!priv_fdir_flow)
6830                 return 0;
6831         LIST_REMOVE(priv_fdir_flow, next);
6832         flow_idx = priv_fdir_flow->rix_flow;
6833         flow_list_destroy(dev, &priv->flows, flow_idx);
6834         mlx5_free(priv_fdir_flow->fdir);
6835         mlx5_free(priv_fdir_flow);
6836         DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
6837                 dev->data->port_id, flow_idx);
6838         return 0;
6839 }
6840
6841 /**
6842  * Update queue for specific filter.
6843  *
6844  * @param dev
6845  *   Pointer to Ethernet device.
6846  * @param fdir_filter
6847  *   Filter to be updated.
6848  *
6849  * @return
6850  *   0 on success, a negative errno value otherwise and rte_errno is set.
6851  */
6852 static int
6853 flow_fdir_filter_update(struct rte_eth_dev *dev,
6854                         const struct rte_eth_fdir_filter *fdir_filter)
6855 {
6856         int ret;
6857
6858         ret = flow_fdir_filter_delete(dev, fdir_filter);
6859         if (ret)
6860                 return ret;
6861         return flow_fdir_filter_add(dev, fdir_filter);
6862 }
6863
6864 /**
6865  * Flush all filters.
6866  *
6867  * @param dev
6868  *   Pointer to Ethernet device.
6869  */
6870 static void
6871 flow_fdir_filter_flush(struct rte_eth_dev *dev)
6872 {
6873         struct mlx5_priv *priv = dev->data->dev_private;
6874         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
6875
6876         while (!LIST_EMPTY(&priv->fdir_flows)) {
6877                 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
6878                 LIST_REMOVE(priv_fdir_flow, next);
6879                 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
6880                 mlx5_free(priv_fdir_flow->fdir);
6881                 mlx5_free(priv_fdir_flow);
6882         }
6883 }
6884
6885 /**
6886  * Get flow director information.
6887  *
6888  * @param dev
6889  *   Pointer to Ethernet device.
6890  * @param[out] fdir_info
6891  *   Resulting flow director information.
6892  */
6893 static void
6894 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
6895 {
6896         struct rte_eth_fdir_masks *mask =
6897                 &dev->data->dev_conf.fdir_conf.mask;
6898
6899         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
6900         fdir_info->guarant_spc = 0;
6901         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
6902         fdir_info->max_flexpayload = 0;
6903         fdir_info->flow_types_mask[0] = 0;
6904         fdir_info->flex_payload_unit = 0;
6905         fdir_info->max_flex_payload_segment_num = 0;
6906         fdir_info->flex_payload_limit = 0;
6907         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
6908 }
6909
6910 /**
6911  * Deal with flow director operations.
6912  *
6913  * @param dev
6914  *   Pointer to Ethernet device.
6915  * @param filter_op
6916  *   Operation to perform.
6917  * @param arg
6918  *   Pointer to operation-specific structure.
6919  *
6920  * @return
6921  *   0 on success, a negative errno value otherwise and rte_errno is set.
6922  */
6923 static int
6924 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
6925                     void *arg)
6926 {
6927         enum rte_fdir_mode fdir_mode =
6928                 dev->data->dev_conf.fdir_conf.mode;
6929
6930         if (filter_op == RTE_ETH_FILTER_NOP)
6931                 return 0;
6932         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
6933             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
6934                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
6935                         dev->data->port_id, fdir_mode);
6936                 rte_errno = EINVAL;
6937                 return -rte_errno;
6938         }
6939         switch (filter_op) {
6940         case RTE_ETH_FILTER_ADD:
6941                 return flow_fdir_filter_add(dev, arg);
6942         case RTE_ETH_FILTER_UPDATE:
6943                 return flow_fdir_filter_update(dev, arg);
6944         case RTE_ETH_FILTER_DELETE:
6945                 return flow_fdir_filter_delete(dev, arg);
6946         case RTE_ETH_FILTER_FLUSH:
6947                 flow_fdir_filter_flush(dev);
6948                 break;
6949         case RTE_ETH_FILTER_INFO:
6950                 flow_fdir_info_get(dev, arg);
6951                 break;
6952         default:
6953                 DRV_LOG(DEBUG, "port %u unknown operation %u",
6954                         dev->data->port_id, filter_op);
6955                 rte_errno = EINVAL;
6956                 return -rte_errno;
6957         }
6958         return 0;
6959 }
6960
6961 /**
6962  * Manage filter operations.
6963  *
6964  * @param dev
6965  *   Pointer to Ethernet device structure.
6966  * @param filter_type
6967  *   Filter type.
6968  * @param filter_op
6969  *   Operation to perform.
6970  * @param arg
6971  *   Pointer to operation-specific structure.
6972  *
6973  * @return
6974  *   0 on success, a negative errno value otherwise and rte_errno is set.
6975  */
6976 int
6977 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6978                      enum rte_filter_type filter_type,
6979                      enum rte_filter_op filter_op,
6980                      void *arg)
6981 {
6982         switch (filter_type) {
6983         case RTE_ETH_FILTER_GENERIC:
6984                 if (filter_op != RTE_ETH_FILTER_GET) {
6985                         rte_errno = EINVAL;
6986                         return -rte_errno;
6987                 }
6988                 *(const void **)arg = &mlx5_flow_ops;
6989                 return 0;
6990         case RTE_ETH_FILTER_FDIR:
6991                 return flow_fdir_ctrl_func(dev, filter_op, arg);
6992         default:
6993                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6994                         dev->data->port_id, filter_type);
6995                 rte_errno = ENOTSUP;
6996                 return -rte_errno;
6997         }
6998         return 0;
6999 }
7000
7001 /**
7002  * Create the needed meter and suffix tables.
7003  *
7004  * @param[in] dev
7005  *   Pointer to Ethernet device.
7006  * @param[in] fm
7007  *   Pointer to the flow meter.
7008  *
7009  * @return
7010  *   Pointer to table set on success, NULL otherwise.
7011  */
7012 struct mlx5_meter_domains_infos *
7013 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7014                           const struct mlx5_flow_meter *fm)
7015 {
7016         const struct mlx5_flow_driver_ops *fops;
7017
7018         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7019         return fops->create_mtr_tbls(dev, fm);
7020 }
7021
7022 /**
7023  * Destroy the meter table set.
7024  *
7025  * @param[in] dev
7026  *   Pointer to Ethernet device.
7027  * @param[in] tbl
7028  *   Pointer to the meter table set.
7029  *
7030  * @return
7031  *   0 on success.
7032  */
7033 int
7034 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7035                            struct mlx5_meter_domains_infos *tbls)
7036 {
7037         const struct mlx5_flow_driver_ops *fops;
7038
7039         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7040         return fops->destroy_mtr_tbls(dev, tbls);
7041 }
7042
7043 /**
7044  * Create policer rules.
7045  *
7046  * @param[in] dev
7047  *   Pointer to Ethernet device.
7048  * @param[in] fm
7049  *   Pointer to flow meter structure.
7050  * @param[in] attr
7051  *   Pointer to flow attributes.
7052  *
7053  * @return
7054  *   0 on success, -1 otherwise.
7055  */
7056 int
7057 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
7058                                struct mlx5_flow_meter *fm,
7059                                const struct rte_flow_attr *attr)
7060 {
7061         const struct mlx5_flow_driver_ops *fops;
7062
7063         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7064         return fops->create_policer_rules(dev, fm, attr);
7065 }
7066
7067 /**
7068  * Destroy policer rules.
7069  *
7070  * @param[in] fm
7071  *   Pointer to flow meter structure.
7072  * @param[in] attr
7073  *   Pointer to flow attributes.
7074  *
7075  * @return
7076  *   0 on success, -1 otherwise.
7077  */
7078 int
7079 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
7080                                 struct mlx5_flow_meter *fm,
7081                                 const struct rte_flow_attr *attr)
7082 {
7083         const struct mlx5_flow_driver_ops *fops;
7084
7085         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7086         return fops->destroy_policer_rules(dev, fm, attr);
7087 }
7088
7089 /**
7090  * Allocate a counter.
7091  *
7092  * @param[in] dev
7093  *   Pointer to Ethernet device structure.
7094  *
7095  * @return
7096  *   Index to allocated counter  on success, 0 otherwise.
7097  */
7098 uint32_t
7099 mlx5_counter_alloc(struct rte_eth_dev *dev)
7100 {
7101         const struct mlx5_flow_driver_ops *fops;
7102         struct rte_flow_attr attr = { .transfer = 0 };
7103
7104         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7105                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7106                 return fops->counter_alloc(dev);
7107         }
7108         DRV_LOG(ERR,
7109                 "port %u counter allocate is not supported.",
7110                  dev->data->port_id);
7111         return 0;
7112 }
7113
7114 /**
7115  * Free a counter.
7116  *
7117  * @param[in] dev
7118  *   Pointer to Ethernet device structure.
7119  * @param[in] cnt
7120  *   Index to counter to be free.
7121  */
7122 void
7123 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7124 {
7125         const struct mlx5_flow_driver_ops *fops;
7126         struct rte_flow_attr attr = { .transfer = 0 };
7127
7128         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7129                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7130                 fops->counter_free(dev, cnt);
7131                 return;
7132         }
7133         DRV_LOG(ERR,
7134                 "port %u counter free is not supported.",
7135                  dev->data->port_id);
7136 }
7137
7138 /**
7139  * Query counter statistics.
7140  *
7141  * @param[in] dev
7142  *   Pointer to Ethernet device structure.
7143  * @param[in] cnt
7144  *   Index to counter to query.
7145  * @param[in] clear
7146  *   Set to clear counter statistics.
7147  * @param[out] pkts
7148  *   The counter hits packets number to save.
7149  * @param[out] bytes
7150  *   The counter hits bytes number to save.
7151  *
7152  * @return
7153  *   0 on success, a negative errno value otherwise.
7154  */
7155 int
7156 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7157                    bool clear, uint64_t *pkts, uint64_t *bytes)
7158 {
7159         const struct mlx5_flow_driver_ops *fops;
7160         struct rte_flow_attr attr = { .transfer = 0 };
7161
7162         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7163                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7164                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7165         }
7166         DRV_LOG(ERR,
7167                 "port %u counter query is not supported.",
7168                  dev->data->port_id);
7169         return -ENOTSUP;
7170 }
7171
7172 /**
7173  * Allocate a new memory for the counter values wrapped by all the needed
7174  * management.
7175  *
7176  * @param[in] sh
7177  *   Pointer to mlx5_dev_ctx_shared object.
7178  *
7179  * @return
7180  *   0 on success, a negative errno value otherwise.
7181  */
7182 static int
7183 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7184 {
7185         struct mlx5_devx_mkey_attr mkey_attr;
7186         struct mlx5_counter_stats_mem_mng *mem_mng;
7187         volatile struct flow_counter_stats *raw_data;
7188         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7189         int size = (sizeof(struct flow_counter_stats) *
7190                         MLX5_COUNTERS_PER_POOL +
7191                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7192                         sizeof(struct mlx5_counter_stats_mem_mng);
7193         size_t pgsize = rte_mem_page_size();
7194         uint8_t *mem;
7195         int i;
7196
7197         if (pgsize == (size_t)-1) {
7198                 DRV_LOG(ERR, "Failed to get mem page size");
7199                 rte_errno = ENOMEM;
7200                 return -ENOMEM;
7201         }
7202         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7203         if (!mem) {
7204                 rte_errno = ENOMEM;
7205                 return -ENOMEM;
7206         }
7207         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7208         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7209         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
7210                                                  IBV_ACCESS_LOCAL_WRITE);
7211         if (!mem_mng->umem) {
7212                 rte_errno = errno;
7213                 mlx5_free(mem);
7214                 return -rte_errno;
7215         }
7216         mkey_attr.addr = (uintptr_t)mem;
7217         mkey_attr.size = size;
7218         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7219         mkey_attr.pd = sh->pdn;
7220         mkey_attr.log_entity_size = 0;
7221         mkey_attr.pg_access = 0;
7222         mkey_attr.klm_array = NULL;
7223         mkey_attr.klm_num = 0;
7224         mkey_attr.relaxed_ordering = sh->cmng.relaxed_ordering;
7225         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7226         if (!mem_mng->dm) {
7227                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
7228                 rte_errno = errno;
7229                 mlx5_free(mem);
7230                 return -rte_errno;
7231         }
7232         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7233         raw_data = (volatile struct flow_counter_stats *)mem;
7234         for (i = 0; i < raws_n; ++i) {
7235                 mem_mng->raws[i].mem_mng = mem_mng;
7236                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7237         }
7238         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7239                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7240                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7241                                  next);
7242         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7243         sh->cmng.mem_mng = mem_mng;
7244         return 0;
7245 }
7246
7247 /**
7248  * Set the statistic memory to the new counter pool.
7249  *
7250  * @param[in] sh
7251  *   Pointer to mlx5_dev_ctx_shared object.
7252  * @param[in] pool
7253  *   Pointer to the pool to set the statistic memory.
7254  *
7255  * @return
7256  *   0 on success, a negative errno value otherwise.
7257  */
7258 static int
7259 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7260                                struct mlx5_flow_counter_pool *pool)
7261 {
7262         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7263         /* Resize statistic memory once used out. */
7264         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7265             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7266                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7267                 return -1;
7268         }
7269         rte_spinlock_lock(&pool->sl);
7270         pool->raw = cmng->mem_mng->raws + pool->index %
7271                     MLX5_CNT_CONTAINER_RESIZE;
7272         rte_spinlock_unlock(&pool->sl);
7273         pool->raw_hw = NULL;
7274         return 0;
7275 }
7276
7277 #define MLX5_POOL_QUERY_FREQ_US 1000000
7278
7279 /**
7280  * Set the periodic procedure for triggering asynchronous batch queries for all
7281  * the counter pools.
7282  *
7283  * @param[in] sh
7284  *   Pointer to mlx5_dev_ctx_shared object.
7285  */
7286 void
7287 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7288 {
7289         uint32_t pools_n, us;
7290
7291         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7292         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7293         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7294         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7295                 sh->cmng.query_thread_on = 0;
7296                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7297         } else {
7298                 sh->cmng.query_thread_on = 1;
7299         }
7300 }
7301
7302 /**
7303  * The periodic procedure for triggering asynchronous batch queries for all the
7304  * counter pools. This function is probably called by the host thread.
7305  *
7306  * @param[in] arg
7307  *   The parameter for the alarm process.
7308  */
7309 void
7310 mlx5_flow_query_alarm(void *arg)
7311 {
7312         struct mlx5_dev_ctx_shared *sh = arg;
7313         int ret;
7314         uint16_t pool_index = sh->cmng.pool_index;
7315         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7316         struct mlx5_flow_counter_pool *pool;
7317         uint16_t n_valid;
7318
7319         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7320                 goto set_alarm;
7321         rte_spinlock_lock(&cmng->pool_update_sl);
7322         pool = cmng->pools[pool_index];
7323         n_valid = cmng->n_valid;
7324         rte_spinlock_unlock(&cmng->pool_update_sl);
7325         /* Set the statistic memory to the new created pool. */
7326         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7327                 goto set_alarm;
7328         if (pool->raw_hw)
7329                 /* There is a pool query in progress. */
7330                 goto set_alarm;
7331         pool->raw_hw =
7332                 LIST_FIRST(&sh->cmng.free_stat_raws);
7333         if (!pool->raw_hw)
7334                 /* No free counter statistics raw memory. */
7335                 goto set_alarm;
7336         /*
7337          * Identify the counters released between query trigger and query
7338          * handle more efficiently. The counter released in this gap period
7339          * should wait for a new round of query as the new arrived packets
7340          * will not be taken into account.
7341          */
7342         pool->query_gen++;
7343         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7344                                                MLX5_COUNTERS_PER_POOL,
7345                                                NULL, NULL,
7346                                                pool->raw_hw->mem_mng->dm->id,
7347                                                (void *)(uintptr_t)
7348                                                pool->raw_hw->data,
7349                                                sh->devx_comp,
7350                                                (uint64_t)(uintptr_t)pool);
7351         if (ret) {
7352                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7353                         " %d", pool->min_dcs->id);
7354                 pool->raw_hw = NULL;
7355                 goto set_alarm;
7356         }
7357         LIST_REMOVE(pool->raw_hw, next);
7358         sh->cmng.pending_queries++;
7359         pool_index++;
7360         if (pool_index >= n_valid)
7361                 pool_index = 0;
7362 set_alarm:
7363         sh->cmng.pool_index = pool_index;
7364         mlx5_set_query_alarm(sh);
7365 }
7366
7367 /**
7368  * Check and callback event for new aged flow in the counter pool
7369  *
7370  * @param[in] sh
7371  *   Pointer to mlx5_dev_ctx_shared object.
7372  * @param[in] pool
7373  *   Pointer to Current counter pool.
7374  */
7375 static void
7376 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7377                    struct mlx5_flow_counter_pool *pool)
7378 {
7379         struct mlx5_priv *priv;
7380         struct mlx5_flow_counter *cnt;
7381         struct mlx5_age_info *age_info;
7382         struct mlx5_age_param *age_param;
7383         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7384         struct mlx5_counter_stats_raw *prev = pool->raw;
7385         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7386         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7387         uint16_t expected = AGE_CANDIDATE;
7388         uint32_t i;
7389
7390         pool->time_of_last_age_check = curr_time;
7391         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7392                 cnt = MLX5_POOL_GET_CNT(pool, i);
7393                 age_param = MLX5_CNT_TO_AGE(cnt);
7394                 if (__atomic_load_n(&age_param->state,
7395                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7396                         continue;
7397                 if (cur->data[i].hits != prev->data[i].hits) {
7398                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7399                                          __ATOMIC_RELAXED);
7400                         continue;
7401                 }
7402                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7403                                        time_delta,
7404                                        __ATOMIC_RELAXED) <= age_param->timeout)
7405                         continue;
7406                 /**
7407                  * Hold the lock first, or if between the
7408                  * state AGE_TMOUT and tailq operation the
7409                  * release happened, the release procedure
7410                  * may delete a non-existent tailq node.
7411                  */
7412                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7413                 age_info = GET_PORT_AGE_INFO(priv);
7414                 rte_spinlock_lock(&age_info->aged_sl);
7415                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7416                                                 AGE_TMOUT, false,
7417                                                 __ATOMIC_RELAXED,
7418                                                 __ATOMIC_RELAXED)) {
7419                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7420                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7421                 }
7422                 rte_spinlock_unlock(&age_info->aged_sl);
7423         }
7424         for (i = 0; i < sh->max_port; i++) {
7425                 age_info = &sh->port[i].age_info;
7426                 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW))
7427                         continue;
7428                 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER))
7429                         rte_eth_dev_callback_process
7430                                 (&rte_eth_devices[sh->port[i].devx_ih_port_id],
7431                                 RTE_ETH_EVENT_FLOW_AGED, NULL);
7432                 age_info->flags = 0;
7433         }
7434 }
7435
7436 /**
7437  * Handler for the HW respond about ready values from an asynchronous batch
7438  * query. This function is probably called by the host thread.
7439  *
7440  * @param[in] sh
7441  *   The pointer to the shared device context.
7442  * @param[in] async_id
7443  *   The Devx async ID.
7444  * @param[in] status
7445  *   The status of the completion.
7446  */
7447 void
7448 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7449                                   uint64_t async_id, int status)
7450 {
7451         struct mlx5_flow_counter_pool *pool =
7452                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7453         struct mlx5_counter_stats_raw *raw_to_free;
7454         uint8_t query_gen = pool->query_gen ^ 1;
7455         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7456         enum mlx5_counter_type cnt_type =
7457                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7458                                 MLX5_COUNTER_TYPE_ORIGIN;
7459
7460         if (unlikely(status)) {
7461                 raw_to_free = pool->raw_hw;
7462         } else {
7463                 raw_to_free = pool->raw;
7464                 if (pool->is_aged)
7465                         mlx5_flow_aging_check(sh, pool);
7466                 rte_spinlock_lock(&pool->sl);
7467                 pool->raw = pool->raw_hw;
7468                 rte_spinlock_unlock(&pool->sl);
7469                 /* Be sure the new raw counters data is updated in memory. */
7470                 rte_io_wmb();
7471                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7472                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7473                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7474                                      &pool->counters[query_gen], next);
7475                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7476                 }
7477         }
7478         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7479         pool->raw_hw = NULL;
7480         sh->cmng.pending_queries--;
7481 }
7482
7483 static const struct mlx5_flow_tbl_data_entry  *
7484 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
7485 {
7486         struct mlx5_priv *priv = dev->data->dev_private;
7487         struct mlx5_dev_ctx_shared *sh = priv->sh;
7488         struct mlx5_hlist_entry *he;
7489         union tunnel_offload_mark mbits = { .val = mark };
7490         union mlx5_flow_tbl_key table_key = {
7491                 {
7492                         .table_id = tunnel_id_to_flow_tbl(mbits.table_id),
7493                         .reserved = 0,
7494                         .domain = !!mbits.transfer,
7495                         .direction = 0,
7496                 }
7497         };
7498         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
7499         return he ?
7500                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
7501 }
7502
7503 static uint32_t
7504 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
7505                                 const struct mlx5_flow_tunnel *tunnel,
7506                                 uint32_t group, uint32_t *table,
7507                                 struct rte_flow_error *error)
7508 {
7509         struct mlx5_hlist_entry *he;
7510         struct tunnel_tbl_entry *tte;
7511         union tunnel_tbl_key key = {
7512                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
7513                 .group = group
7514         };
7515         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
7516         struct mlx5_hlist *group_hash;
7517
7518         group_hash = tunnel ? tunnel->groups : thub->groups;
7519         he = mlx5_hlist_lookup(group_hash, key.val);
7520         if (!he) {
7521                 int ret;
7522                 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
7523                                   sizeof(*tte), 0,
7524                                   SOCKET_ID_ANY);
7525                 if (!tte)
7526                         goto err;
7527                 tte->hash.key = key.val;
7528                 ret = mlx5_flow_id_get(thub->table_ids, &tte->flow_table);
7529                 if (ret) {
7530                         mlx5_free(tte);
7531                         goto err;
7532                 }
7533                 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
7534                 mlx5_hlist_insert(group_hash, &tte->hash);
7535         } else {
7536                 tte = container_of(he, typeof(*tte), hash);
7537         }
7538         *table = tte->flow_table;
7539         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
7540                 dev->data->port_id, key.tunnel_id, group, *table);
7541         return 0;
7542
7543 err:
7544         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7545                                   NULL, "tunnel group index not supported");
7546 }
7547
7548 static int
7549 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7550                     struct flow_grp_info grp_info, struct rte_flow_error *error)
7551 {
7552         if (grp_info.transfer && grp_info.external && grp_info.fdb_def_rule) {
7553                 if (group == UINT32_MAX)
7554                         return rte_flow_error_set
7555                                                 (error, EINVAL,
7556                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7557                                                  NULL,
7558                                                  "group index not supported");
7559                 *table = group + 1;
7560         } else {
7561                 *table = group;
7562         }
7563         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7564         return 0;
7565 }
7566
7567 /**
7568  * Translate the rte_flow group index to HW table value.
7569  *
7570  * If tunnel offload is disabled, all group ids converted to flow table
7571  * id using the standard method.
7572  * If tunnel offload is enabled, group id can be converted using the
7573  * standard or tunnel conversion method. Group conversion method
7574  * selection depends on flags in `grp_info` parameter:
7575  * - Internal (grp_info.external == 0) groups conversion uses the
7576  *   standard method.
7577  * - Group ids in JUMP action converted with the tunnel conversion.
7578  * - Group id in rule attribute conversion depends on a rule type and
7579  *   group id value:
7580  *   ** non zero group attributes converted with the tunnel method
7581  *   ** zero group attribute in non-tunnel rule is converted using the
7582  *      standard method - there's only one root table
7583  *   ** zero group attribute in steer tunnel rule is converted with the
7584  *      standard method - single root table
7585  *   ** zero group attribute in match tunnel rule is a special OvS
7586  *      case: that value is used for portability reasons. That group
7587  *      id is converted with the tunnel conversion method.
7588  *
7589  * @param[in] dev
7590  *   Port device
7591  * @param[in] tunnel
7592  *   PMD tunnel offload object
7593  * @param[in] group
7594  *   rte_flow group index value.
7595  * @param[out] table
7596  *   HW table value.
7597  * @param[in] grp_info
7598  *   flags used for conversion
7599  * @param[out] error
7600  *   Pointer to error structure.
7601  *
7602  * @return
7603  *   0 on success, a negative errno value otherwise and rte_errno is set.
7604  */
7605 int
7606 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7607                          const struct mlx5_flow_tunnel *tunnel,
7608                          uint32_t group, uint32_t *table,
7609                          struct flow_grp_info grp_info,
7610                          struct rte_flow_error *error)
7611 {
7612         int ret;
7613         bool standard_translation;
7614
7615         if (grp_info.external && group < MLX5_MAX_TABLES_EXTERNAL)
7616                 group *= MLX5_FLOW_TABLE_FACTOR;
7617         if (is_tunnel_offload_active(dev)) {
7618                 standard_translation = !grp_info.external ||
7619                                         grp_info.std_tbl_fix;
7620         } else {
7621                 standard_translation = true;
7622         }
7623         DRV_LOG(DEBUG,
7624                 "port %u group=%#x transfer=%d external=%d fdb_def_rule=%d translate=%s",
7625                 dev->data->port_id, group, grp_info.transfer,
7626                 grp_info.external, grp_info.fdb_def_rule,
7627                 standard_translation ? "STANDARD" : "TUNNEL");
7628         if (standard_translation)
7629                 ret = flow_group_to_table(dev->data->port_id, group, table,
7630                                           grp_info, error);
7631         else
7632                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7633                                                       table, error);
7634
7635         return ret;
7636 }
7637
7638 /**
7639  * Discover availability of metadata reg_c's.
7640  *
7641  * Iteratively use test flows to check availability.
7642  *
7643  * @param[in] dev
7644  *   Pointer to the Ethernet device structure.
7645  *
7646  * @return
7647  *   0 on success, a negative errno value otherwise and rte_errno is set.
7648  */
7649 int
7650 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7651 {
7652         struct mlx5_priv *priv = dev->data->dev_private;
7653         struct mlx5_dev_config *config = &priv->config;
7654         enum modify_reg idx;
7655         int n = 0;
7656
7657         /* reg_c[0] and reg_c[1] are reserved. */
7658         config->flow_mreg_c[n++] = REG_C_0;
7659         config->flow_mreg_c[n++] = REG_C_1;
7660         /* Discover availability of other reg_c's. */
7661         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7662                 struct rte_flow_attr attr = {
7663                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7664                         .priority = MLX5_FLOW_PRIO_RSVD,
7665                         .ingress = 1,
7666                 };
7667                 struct rte_flow_item items[] = {
7668                         [0] = {
7669                                 .type = RTE_FLOW_ITEM_TYPE_END,
7670                         },
7671                 };
7672                 struct rte_flow_action actions[] = {
7673                         [0] = {
7674                                 .type = (enum rte_flow_action_type)
7675                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7676                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7677                                         .src = REG_C_1,
7678                                         .dst = idx,
7679                                 },
7680                         },
7681                         [1] = {
7682                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7683                                 .conf = &(struct rte_flow_action_jump){
7684                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7685                                 },
7686                         },
7687                         [2] = {
7688                                 .type = RTE_FLOW_ACTION_TYPE_END,
7689                         },
7690                 };
7691                 uint32_t flow_idx;
7692                 struct rte_flow *flow;
7693                 struct rte_flow_error error;
7694
7695                 if (!config->dv_flow_en)
7696                         break;
7697                 /* Create internal flow, validation skips copy action. */
7698                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7699                                             actions, false, &error);
7700                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7701                                       flow_idx);
7702                 if (!flow)
7703                         continue;
7704                 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
7705                         config->flow_mreg_c[n++] = idx;
7706                 flow_list_destroy(dev, NULL, flow_idx);
7707         }
7708         for (; n < MLX5_MREG_C_NUM; ++n)
7709                 config->flow_mreg_c[n] = REG_NON;
7710         return 0;
7711 }
7712
7713 /**
7714  * Dump flow raw hw data to file
7715  *
7716  * @param[in] dev
7717  *    The pointer to Ethernet device.
7718  * @param[in] file
7719  *   A pointer to a file for output.
7720  * @param[out] error
7721  *   Perform verbose error reporting if not NULL. PMDs initialize this
7722  *   structure in case of error only.
7723  * @return
7724  *   0 on success, a nagative value otherwise.
7725  */
7726 int
7727 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
7728                    FILE *file,
7729                    struct rte_flow_error *error __rte_unused)
7730 {
7731         struct mlx5_priv *priv = dev->data->dev_private;
7732         struct mlx5_dev_ctx_shared *sh = priv->sh;
7733
7734         if (!priv->config.dv_flow_en) {
7735                 if (fputs("device dv flow disabled\n", file) <= 0)
7736                         return -errno;
7737                 return -ENOTSUP;
7738         }
7739         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
7740                                        sh->tx_domain, file);
7741 }
7742
7743 /**
7744  * Get aged-out flows.
7745  *
7746  * @param[in] dev
7747  *   Pointer to the Ethernet device structure.
7748  * @param[in] context
7749  *   The address of an array of pointers to the aged-out flows contexts.
7750  * @param[in] nb_countexts
7751  *   The length of context array pointers.
7752  * @param[out] error
7753  *   Perform verbose error reporting if not NULL. Initialized in case of
7754  *   error only.
7755  *
7756  * @return
7757  *   how many contexts get in success, otherwise negative errno value.
7758  *   if nb_contexts is 0, return the amount of all aged contexts.
7759  *   if nb_contexts is not 0 , return the amount of aged flows reported
7760  *   in the context array.
7761  */
7762 int
7763 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
7764                         uint32_t nb_contexts, struct rte_flow_error *error)
7765 {
7766         const struct mlx5_flow_driver_ops *fops;
7767         struct rte_flow_attr attr = { .transfer = 0 };
7768
7769         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7770                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7771                 return fops->get_aged_flows(dev, contexts, nb_contexts,
7772                                                     error);
7773         }
7774         DRV_LOG(ERR,
7775                 "port %u get aged flows is not supported.",
7776                  dev->data->port_id);
7777         return -ENOTSUP;
7778 }
7779
7780 /* Wrapper for driver action_validate op callback */
7781 static int
7782 flow_drv_action_validate(struct rte_eth_dev *dev,
7783                          const struct rte_flow_shared_action_conf *conf,
7784                          const struct rte_flow_action *action,
7785                          const struct mlx5_flow_driver_ops *fops,
7786                          struct rte_flow_error *error)
7787 {
7788         static const char err_msg[] = "shared action validation unsupported";
7789
7790         if (!fops->action_validate) {
7791                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7792                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7793                                    NULL, err_msg);
7794                 return -rte_errno;
7795         }
7796         return fops->action_validate(dev, conf, action, error);
7797 }
7798
7799 /**
7800  * Destroys the shared action by handle.
7801  *
7802  * @param dev
7803  *   Pointer to Ethernet device structure.
7804  * @param[in] action
7805  *   Handle for the shared action to be destroyed.
7806  * @param[out] error
7807  *   Perform verbose error reporting if not NULL. PMDs initialize this
7808  *   structure in case of error only.
7809  *
7810  * @return
7811  *   0 on success, a negative errno value otherwise and rte_errno is set.
7812  *
7813  * @note: wrapper for driver action_create op callback.
7814  */
7815 static int
7816 mlx5_shared_action_destroy(struct rte_eth_dev *dev,
7817                            struct rte_flow_shared_action *action,
7818                            struct rte_flow_error *error)
7819 {
7820         static const char err_msg[] = "shared action destruction unsupported";
7821         struct rte_flow_attr attr = { .transfer = 0 };
7822         const struct mlx5_flow_driver_ops *fops =
7823                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7824
7825         if (!fops->action_destroy) {
7826                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7827                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7828                                    NULL, err_msg);
7829                 return -rte_errno;
7830         }
7831         return fops->action_destroy(dev, action, error);
7832 }
7833
7834 /* Wrapper for driver action_destroy op callback */
7835 static int
7836 flow_drv_action_update(struct rte_eth_dev *dev,
7837                        struct rte_flow_shared_action *action,
7838                        const void *action_conf,
7839                        const struct mlx5_flow_driver_ops *fops,
7840                        struct rte_flow_error *error)
7841 {
7842         static const char err_msg[] = "shared action update unsupported";
7843
7844         if (!fops->action_update) {
7845                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7846                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7847                                    NULL, err_msg);
7848                 return -rte_errno;
7849         }
7850         return fops->action_update(dev, action, action_conf, error);
7851 }
7852
7853 /**
7854  * Create shared action for reuse in multiple flow rules.
7855  *
7856  * @param dev
7857  *   Pointer to Ethernet device structure.
7858  * @param[in] action
7859  *   Action configuration for shared action creation.
7860  * @param[out] error
7861  *   Perform verbose error reporting if not NULL. PMDs initialize this
7862  *   structure in case of error only.
7863  * @return
7864  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
7865  */
7866 static struct rte_flow_shared_action *
7867 mlx5_shared_action_create(struct rte_eth_dev *dev,
7868                           const struct rte_flow_shared_action_conf *conf,
7869                           const struct rte_flow_action *action,
7870                           struct rte_flow_error *error)
7871 {
7872         static const char err_msg[] = "shared action creation unsupported";
7873         struct rte_flow_attr attr = { .transfer = 0 };
7874         const struct mlx5_flow_driver_ops *fops =
7875                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7876
7877         if (flow_drv_action_validate(dev, conf, action, fops, error))
7878                 return NULL;
7879         if (!fops->action_create) {
7880                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7881                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7882                                    NULL, err_msg);
7883                 return NULL;
7884         }
7885         return fops->action_create(dev, conf, action, error);
7886 }
7887
7888 /**
7889  * Updates inplace the shared action configuration pointed by *action* handle
7890  * with the configuration provided as *action* argument.
7891  * The update of the shared action configuration effects all flow rules reusing
7892  * the action via handle.
7893  *
7894  * @param dev
7895  *   Pointer to Ethernet device structure.
7896  * @param[in] shared_action
7897  *   Handle for the shared action to be updated.
7898  * @param[in] action
7899  *   Action specification used to modify the action pointed by handle.
7900  *   *action* should be of same type with the action pointed by the *action*
7901  *   handle argument, otherwise considered as invalid.
7902  * @param[out] error
7903  *   Perform verbose error reporting if not NULL. PMDs initialize this
7904  *   structure in case of error only.
7905  *
7906  * @return
7907  *   0 on success, a negative errno value otherwise and rte_errno is set.
7908  */
7909 static int
7910 mlx5_shared_action_update(struct rte_eth_dev *dev,
7911                 struct rte_flow_shared_action *shared_action,
7912                 const struct rte_flow_action *action,
7913                 struct rte_flow_error *error)
7914 {
7915         struct rte_flow_attr attr = { .transfer = 0 };
7916         const struct mlx5_flow_driver_ops *fops =
7917                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
7918         int ret;
7919
7920         switch (shared_action->type) {
7921         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
7922                 if (action->type != RTE_FLOW_ACTION_TYPE_RSS) {
7923                         return rte_flow_error_set(error, EINVAL,
7924                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7925                                                   NULL,
7926                                                   "update action type invalid");
7927                 }
7928                 ret = flow_drv_action_validate(dev, NULL, action, fops, error);
7929                 if (ret)
7930                         return ret;
7931                 return flow_drv_action_update(dev, shared_action, action->conf,
7932                                               fops, error);
7933         default:
7934                 return rte_flow_error_set(error, ENOTSUP,
7935                                           RTE_FLOW_ERROR_TYPE_ACTION,
7936                                           NULL,
7937                                           "action type not supported");
7938         }
7939 }
7940
7941 /**
7942  * Query the shared action by handle.
7943  *
7944  * This function allows retrieving action-specific data such as counters.
7945  * Data is gathered by special action which may be present/referenced in
7946  * more than one flow rule definition.
7947  *
7948  * \see RTE_FLOW_ACTION_TYPE_COUNT
7949  *
7950  * @param dev
7951  *   Pointer to Ethernet device structure.
7952  * @param[in] action
7953  *   Handle for the shared action to query.
7954  * @param[in, out] data
7955  *   Pointer to storage for the associated query data type.
7956  * @param[out] error
7957  *   Perform verbose error reporting if not NULL. PMDs initialize this
7958  *   structure in case of error only.
7959  *
7960  * @return
7961  *   0 on success, a negative errno value otherwise and rte_errno is set.
7962  */
7963 static int
7964 mlx5_shared_action_query(struct rte_eth_dev *dev,
7965                          const struct rte_flow_shared_action *action,
7966                          void *data,
7967                          struct rte_flow_error *error)
7968 {
7969         (void)dev;
7970         switch (action->type) {
7971         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
7972                 __atomic_load(&action->refcnt, (uint32_t *)data,
7973                               __ATOMIC_RELAXED);
7974                 return 0;
7975         default:
7976                 return rte_flow_error_set(error, ENOTSUP,
7977                                           RTE_FLOW_ERROR_TYPE_ACTION,
7978                                           NULL,
7979                                           "action type not supported");
7980         }
7981 }
7982
7983 /**
7984  * Destroy all shared actions.
7985  *
7986  * @param dev
7987  *   Pointer to Ethernet device.
7988  *
7989  * @return
7990  *   0 on success, a negative errno value otherwise and rte_errno is set.
7991  */
7992 int
7993 mlx5_shared_action_flush(struct rte_eth_dev *dev)
7994 {
7995         struct rte_flow_error error;
7996         struct mlx5_priv *priv = dev->data->dev_private;
7997         struct rte_flow_shared_action *action;
7998         int ret = 0;
7999
8000         while (!LIST_EMPTY(&priv->shared_actions)) {
8001                 action = LIST_FIRST(&priv->shared_actions);
8002                 ret = mlx5_shared_action_destroy(dev, action, &error);
8003         }
8004         return ret;
8005 }
8006
8007 static void
8008 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8009                       struct mlx5_flow_tunnel *tunnel)
8010 {
8011         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8012         struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids;
8013
8014         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8015                 dev->data->port_id, tunnel->tunnel_id);
8016         RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED));
8017         LIST_REMOVE(tunnel, chain);
8018         mlx5_flow_id_release(id_pool, tunnel->tunnel_id);
8019         mlx5_hlist_destroy(tunnel->groups, NULL, NULL);
8020         mlx5_free(tunnel);
8021 }
8022
8023 static struct mlx5_flow_tunnel *
8024 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8025 {
8026         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8027         struct mlx5_flow_tunnel *tun;
8028
8029         LIST_FOREACH(tun, &thub->tunnels, chain) {
8030                 if (tun->tunnel_id == id)
8031                         break;
8032         }
8033
8034         return tun;
8035 }
8036
8037 static struct mlx5_flow_tunnel *
8038 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8039                           const struct rte_flow_tunnel *app_tunnel)
8040 {
8041         int ret;
8042         struct mlx5_flow_tunnel *tunnel;
8043         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8044         struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids;
8045         uint32_t id;
8046
8047         ret = mlx5_flow_id_get(id_pool, &id);
8048         if (ret)
8049                 return NULL;
8050         /**
8051          * mlx5 flow tunnel is an auxlilary data structure
8052          * It's not part of IO. No need to allocate it from
8053          * huge pages pools dedicated for IO
8054          */
8055         tunnel = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*tunnel),
8056                              0, SOCKET_ID_ANY);
8057         if (!tunnel) {
8058                 mlx5_flow_id_pool_release(id_pool);
8059                 return NULL;
8060         }
8061         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024);
8062         if (!tunnel->groups) {
8063                 mlx5_flow_id_pool_release(id_pool);
8064                 mlx5_free(tunnel);
8065                 return NULL;
8066         }
8067         /* initiate new PMD tunnel */
8068         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8069         tunnel->tunnel_id = id;
8070         tunnel->action.type = (typeof(tunnel->action.type))
8071                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8072         tunnel->action.conf = tunnel;
8073         tunnel->item.type = (typeof(tunnel->item.type))
8074                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8075         tunnel->item.spec = tunnel;
8076         tunnel->item.last = NULL;
8077         tunnel->item.mask = NULL;
8078
8079         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8080                 dev->data->port_id, tunnel->tunnel_id);
8081
8082         return tunnel;
8083 }
8084
8085 static int
8086 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8087                      const struct rte_flow_tunnel *app_tunnel,
8088                      struct mlx5_flow_tunnel **tunnel)
8089 {
8090         int ret;
8091         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8092         struct mlx5_flow_tunnel *tun;
8093
8094         LIST_FOREACH(tun, &thub->tunnels, chain) {
8095                 if (!memcmp(app_tunnel, &tun->app_tunnel,
8096                             sizeof(*app_tunnel))) {
8097                         *tunnel = tun;
8098                         ret = 0;
8099                         break;
8100                 }
8101         }
8102         if (!tun) {
8103                 tun = mlx5_flow_tunnel_allocate(dev, app_tunnel);
8104                 if (tun) {
8105                         LIST_INSERT_HEAD(&thub->tunnels, tun, chain);
8106                         *tunnel = tun;
8107                 } else {
8108                         ret = -ENOMEM;
8109                 }
8110         }
8111         if (tun)
8112                 __atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
8113
8114         return ret;
8115 }
8116
8117 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8118 {
8119         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8120
8121         if (!thub)
8122                 return;
8123         if (!LIST_EMPTY(&thub->tunnels))
8124                 DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
8125         mlx5_flow_id_pool_release(thub->tunnel_ids);
8126         mlx5_flow_id_pool_release(thub->table_ids);
8127         mlx5_hlist_destroy(thub->groups, NULL, NULL);
8128         mlx5_free(thub);
8129 }
8130
8131 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8132 {
8133         int err;
8134         struct mlx5_flow_tunnel_hub *thub;
8135
8136         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8137                            0, SOCKET_ID_ANY);
8138         if (!thub)
8139                 return -ENOMEM;
8140         LIST_INIT(&thub->tunnels);
8141         thub->tunnel_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TUNNELS);
8142         if (!thub->tunnel_ids) {
8143                 err = -rte_errno;
8144                 goto err;
8145         }
8146         thub->table_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TABLES);
8147         if (!thub->table_ids) {
8148                 err = -rte_errno;
8149                 goto err;
8150         }
8151         thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES);
8152         if (!thub->groups) {
8153                 err = -rte_errno;
8154                 goto err;
8155         }
8156         sh->tunnel_hub = thub;
8157
8158         return 0;
8159
8160 err:
8161         if (thub->groups)
8162                 mlx5_hlist_destroy(thub->groups, NULL, NULL);
8163         if (thub->table_ids)
8164                 mlx5_flow_id_pool_release(thub->table_ids);
8165         if (thub->tunnel_ids)
8166                 mlx5_flow_id_pool_release(thub->tunnel_ids);
8167         if (thub)
8168                 mlx5_free(thub);
8169         return err;
8170 }