drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdbool.h>
  12
  13 #include <rte_common.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rxtx.h"
  32
  33 /** Device flow drivers. */
  34 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  35
  36 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  37
  38 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  39         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  40 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  41         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  42 #endif
  43         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  44         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  45 };
  46
  47 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  48 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  49         (const int []){ \
  50                 __VA_ARGS__, 0, \
  51         }
  52
  53 /** Node object of input graph for mlx5_flow_expand_rss(). */
  54 struct mlx5_flow_expand_node {
  55         const int *const next;
  56         /**<
  57          * List of next node indexes. Index 0 is interpreted as a terminator.
  58          */
  59         const enum rte_flow_item_type type;
  60         /**< Pattern item type of current node. */
  61         uint64_t rss_types;
  62         /**<
  63          * RSS types bit-field associated with this node
  64          * (see ETH_RSS_* definitions).
  65          */
  66 };
  67
  68 /** Object returned by mlx5_flow_expand_rss(). */
  69 struct mlx5_flow_expand_rss {
  70         uint32_t entries;
  71         /**< Number of entries @p patterns and @p priorities. */
  72         struct {
  73                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
  74                 uint32_t priority; /**< Priority offset for each expansion. */
  75         } entry[];
  76 };
  77
  78 static enum rte_flow_item_type
  79 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
  80 {
  81         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
  82         uint16_t ether_type = 0;
  83         uint16_t ether_type_m;
  84         uint8_t ip_next_proto = 0;
  85         uint8_t ip_next_proto_m;
  86
  87         if (item == NULL || item->spec == NULL)
  88                 return ret;
  89         switch (item->type) {
  90         case RTE_FLOW_ITEM_TYPE_ETH:
  91                 if (item->mask)
  92                         ether_type_m = ((const struct rte_flow_item_eth *)
  93                                                 (item->mask))->type;
  94                 else
  95                         ether_type_m = rte_flow_item_eth_mask.type;
  96                 if (ether_type_m != RTE_BE16(0xFFFF))
  97                         break;
  98                 ether_type = ((const struct rte_flow_item_eth *)
  99                                 (item->spec))->type;
 100                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 101                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 102                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 103                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 104                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 105                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 106                 else
 107                         ret = RTE_FLOW_ITEM_TYPE_END;
 108                 break;
 109         case RTE_FLOW_ITEM_TYPE_VLAN:
 110                 if (item->mask)
 111                         ether_type_m = ((const struct rte_flow_item_vlan *)
 112                                                 (item->mask))->inner_type;
 113                 else
 114                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 115                 if (ether_type_m != RTE_BE16(0xFFFF))
 116                         break;
 117                 ether_type = ((const struct rte_flow_item_vlan *)
 118                                 (item->spec))->inner_type;
 119                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 120                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 121                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 122                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 123                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 124                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 125                 else
 126                         ret = RTE_FLOW_ITEM_TYPE_END;
 127                 break;
 128         case RTE_FLOW_ITEM_TYPE_IPV4:
 129                 if (item->mask)
 130                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 131                                         (item->mask))->hdr.next_proto_id;
 132                 else
 133                         ip_next_proto_m =
 134                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 135                 if (ip_next_proto_m != 0xFF)
 136                         break;
 137                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 138                                 (item->spec))->hdr.next_proto_id;
 139                 if (ip_next_proto == IPPROTO_UDP)
 140                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 141                 else if (ip_next_proto == IPPROTO_TCP)
 142                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 143                 else if (ip_next_proto == IPPROTO_IP)
 144                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 145                 else if (ip_next_proto == IPPROTO_IPV6)
 146                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 147                 else
 148                         ret = RTE_FLOW_ITEM_TYPE_END;
 149                 break;
 150         case RTE_FLOW_ITEM_TYPE_IPV6:
 151                 if (item->mask)
 152                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 153                                                 (item->mask))->hdr.proto;
 154                 else
 155                         ip_next_proto_m =
 156                                 rte_flow_item_ipv6_mask.hdr.proto;
 157                 if (ip_next_proto_m != 0xFF)
 158                         break;
 159                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 160                                 (item->spec))->hdr.proto;
 161                 if (ip_next_proto == IPPROTO_UDP)
 162                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 163                 else if (ip_next_proto == IPPROTO_TCP)
 164                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 165                 else if (ip_next_proto == IPPROTO_IP)
 166                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 167                 else if (ip_next_proto == IPPROTO_IPV6)
 168                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 169                 else
 170                         ret = RTE_FLOW_ITEM_TYPE_END;
 171                 break;
 172         default:
 173                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 174                 break;
 175         }
 176         return ret;
 177 }
 178
 179 /**
 180  * Expand RSS flows into several possible flows according to the RSS hash
 181  * fields requested and the driver capabilities.
 182  *
 183  * @param[out] buf
 184  *   Buffer to store the result expansion.
 185  * @param[in] size
 186  *   Buffer size in bytes. If 0, @p buf can be NULL.
 187  * @param[in] pattern
 188  *   User flow pattern.
 189  * @param[in] types
 190  *   RSS types to expand (see ETH_RSS_* definitions).
 191  * @param[in] graph
 192  *   Input graph to expand @p pattern according to @p types.
 193  * @param[in] graph_root_index
 194  *   Index of root node in @p graph, typically 0.
 195  *
 196  * @return
 197  *   A positive value representing the size of @p buf in bytes regardless of
 198  *   @p size on success, a negative errno value otherwise and rte_errno is
 199  *   set, the following errors are defined:
 200  *
 201  *   -E2BIG: graph-depth @p graph is too deep.
 202  */
 203 static int
 204 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 205                      const struct rte_flow_item *pattern, uint64_t types,
 206                      const struct mlx5_flow_expand_node graph[],
 207                      int graph_root_index)
 208 {
 209         const int elt_n = 8;
 210         const struct rte_flow_item *item;
 211         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 212         const int *next_node;
 213         const int *stack[elt_n];
 214         int stack_pos = 0;
 215         struct rte_flow_item flow_items[elt_n];
 216         unsigned int i;
 217         size_t lsize;
 218         size_t user_pattern_size = 0;
 219         void *addr = NULL;
 220         const struct mlx5_flow_expand_node *next = NULL;
 221         struct rte_flow_item missed_item;
 222         int missed = 0;
 223         int elt = 0;
 224         const struct rte_flow_item *last_item = NULL;
 225
 226         memset(&missed_item, 0, sizeof(missed_item));
 227         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 228                 elt_n * sizeof(buf->entry[0]);
 229         if (lsize <= size) {
 230                 buf->entry[0].priority = 0;
 231                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 232                 buf->entries = 0;
 233                 addr = buf->entry[0].pattern;
 234         }
 235         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 236                 if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
 237                         last_item = item;
 238                 for (i = 0; node->next && node->next[i]; ++i) {
 239                         next = &graph[node->next[i]];
 240                         if (next->type == item->type)
 241                                 break;
 242                 }
 243                 if (next)
 244                         node = next;
 245                 user_pattern_size += sizeof(*item);
 246         }
 247         user_pattern_size += sizeof(*item); /* Handle END item. */
 248         lsize += user_pattern_size;
 249         /* Copy the user pattern in the first entry of the buffer. */
 250         if (lsize <= size) {
 251                 rte_memcpy(addr, pattern, user_pattern_size);
 252                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 253                 buf->entries = 1;
 254         }
 255         /* Start expanding. */
 256         memset(flow_items, 0, sizeof(flow_items));
 257         user_pattern_size -= sizeof(*item);
 258         /*
 259          * Check if the last valid item has spec set, need complete pattern,
 260          * and the pattern can be used for expansion.
 261          */
 262         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 263         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 264                 /* Item type END indicates expansion is not required. */
 265                 return lsize;
 266         }
 267         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 268                 next = NULL;
 269                 missed = 1;
 270                 for (i = 0; node->next && node->next[i]; ++i) {
 271                         next = &graph[node->next[i]];
 272                         if (next->type == missed_item.type) {
 273                                 flow_items[0].type = missed_item.type;
 274                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 275                                 break;
 276                         }
 277                         next = NULL;
 278                 }
 279         }
 280         if (next && missed) {
 281                 elt = 2; /* missed item + item end. */
 282                 node = next;
 283                 lsize += elt * sizeof(*item) + user_pattern_size;
 284                 if ((node->rss_types & types) && lsize <= size) {
 285                         buf->entry[buf->entries].priority = 1;
 286                         buf->entry[buf->entries].pattern = addr;
 287                         buf->entries++;
 288                         rte_memcpy(addr, buf->entry[0].pattern,
 289                                    user_pattern_size);
 290                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 291                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 292                         addr = (void *)(((uintptr_t)addr) +
 293                                         elt * sizeof(*item));
 294                 }
 295         }
 296         memset(flow_items, 0, sizeof(flow_items));
 297         next_node = node->next;
 298         stack[stack_pos] = next_node;
 299         node = next_node ? &graph[*next_node] : NULL;
 300         while (node) {
 301                 flow_items[stack_pos].type = node->type;
 302                 if (node->rss_types & types) {
 303                         /*
 304                          * compute the number of items to copy from the
 305                          * expansion and copy it.
 306                          * When the stack_pos is 0, there are 1 element in it,
 307                          * plus the addition END item.
 308                          */
 309                         elt = stack_pos + 2;
 310                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 311                         lsize += elt * sizeof(*item) + user_pattern_size;
 312                         if (lsize <= size) {
 313                                 size_t n = elt * sizeof(*item);
 314
 315                                 buf->entry[buf->entries].priority =
 316                                         stack_pos + 1 + missed;
 317                                 buf->entry[buf->entries].pattern = addr;
 318                                 buf->entries++;
 319                                 rte_memcpy(addr, buf->entry[0].pattern,
 320                                            user_pattern_size);
 321                                 addr = (void *)(((uintptr_t)addr) +
 322                                                 user_pattern_size);
 323                                 rte_memcpy(addr, &missed_item,
 324                                            missed * sizeof(*item));
 325                                 addr = (void *)(((uintptr_t)addr) +
 326                                         missed * sizeof(*item));
 327                                 rte_memcpy(addr, flow_items, n);
 328                                 addr = (void *)(((uintptr_t)addr) + n);
 329                         }
 330                 }
 331                 /* Go deeper. */
 332                 if (node->next) {
 333                         next_node = node->next;
 334                         if (stack_pos++ == elt_n) {
 335                                 rte_errno = E2BIG;
 336                                 return -rte_errno;
 337                         }
 338                         stack[stack_pos] = next_node;
 339                 } else if (*(next_node + 1)) {
 340                         /* Follow up with the next possibility. */
 341                         ++next_node;
 342                 } else {
 343                         /* Move to the next path. */
 344                         if (stack_pos)
 345                                 next_node = stack[--stack_pos];
 346                         next_node++;
 347                         stack[stack_pos] = next_node;
 348                 }
 349                 node = *next_node ? &graph[*next_node] : NULL;
 350         };
 351         /* no expanded flows but we have missed item, create one rule for it */
 352         if (buf->entries == 1 && missed != 0) {
 353                 elt = 2;
 354                 lsize += elt * sizeof(*item) + user_pattern_size;
 355                 if (lsize <= size) {
 356                         buf->entry[buf->entries].priority = 1;
 357                         buf->entry[buf->entries].pattern = addr;
 358                         buf->entries++;
 359                         flow_items[0].type = missed_item.type;
 360                         flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 361                         rte_memcpy(addr, buf->entry[0].pattern,
 362                                    user_pattern_size);
 363                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 364                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 365                         addr = (void *)(((uintptr_t)addr) +
 366                                         elt * sizeof(*item));
 367                 }
 368         }
 369         return lsize;
 370 }
 371
 372 enum mlx5_expansion {
 373         MLX5_EXPANSION_ROOT,
 374         MLX5_EXPANSION_ROOT_OUTER,
 375         MLX5_EXPANSION_ROOT_ETH_VLAN,
 376         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 377         MLX5_EXPANSION_OUTER_ETH,
 378         MLX5_EXPANSION_OUTER_ETH_VLAN,
 379         MLX5_EXPANSION_OUTER_VLAN,
 380         MLX5_EXPANSION_OUTER_IPV4,
 381         MLX5_EXPANSION_OUTER_IPV4_UDP,
 382         MLX5_EXPANSION_OUTER_IPV4_TCP,
 383         MLX5_EXPANSION_OUTER_IPV6,
 384         MLX5_EXPANSION_OUTER_IPV6_UDP,
 385         MLX5_EXPANSION_OUTER_IPV6_TCP,
 386         MLX5_EXPANSION_VXLAN,
 387         MLX5_EXPANSION_VXLAN_GPE,
 388         MLX5_EXPANSION_GRE,
 389         MLX5_EXPANSION_MPLS,
 390         MLX5_EXPANSION_ETH,
 391         MLX5_EXPANSION_ETH_VLAN,
 392         MLX5_EXPANSION_VLAN,
 393         MLX5_EXPANSION_IPV4,
 394         MLX5_EXPANSION_IPV4_UDP,
 395         MLX5_EXPANSION_IPV4_TCP,
 396         MLX5_EXPANSION_IPV6,
 397         MLX5_EXPANSION_IPV6_UDP,
 398         MLX5_EXPANSION_IPV6_TCP,
 399 };
 400
 401 /** Supported expansion of items. */
 402 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 403         [MLX5_EXPANSION_ROOT] = {
 404                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 405                                                   MLX5_EXPANSION_IPV4,
 406                                                   MLX5_EXPANSION_IPV6),
 407                 .type = RTE_FLOW_ITEM_TYPE_END,
 408         },
 409         [MLX5_EXPANSION_ROOT_OUTER] = {
 410                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 411                                                   MLX5_EXPANSION_OUTER_IPV4,
 412                                                   MLX5_EXPANSION_OUTER_IPV6),
 413                 .type = RTE_FLOW_ITEM_TYPE_END,
 414         },
 415         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 416                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 417                 .type = RTE_FLOW_ITEM_TYPE_END,
 418         },
 419         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 420                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 421                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 422                 .type = RTE_FLOW_ITEM_TYPE_END,
 423         },
 424         [MLX5_EXPANSION_OUTER_ETH] = {
 425                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 426                                                   MLX5_EXPANSION_OUTER_IPV6,
 427                                                   MLX5_EXPANSION_MPLS),
 428                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 429                 .rss_types = 0,
 430         },
 431         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 432                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 433                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 434                 .rss_types = 0,
 435         },
 436         [MLX5_EXPANSION_OUTER_VLAN] = {
 437                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 438                                                   MLX5_EXPANSION_OUTER_IPV6),
 439                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 440         },
 441         [MLX5_EXPANSION_OUTER_IPV4] = {
 442                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 443                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 444                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 445                          MLX5_EXPANSION_GRE,
 446                          MLX5_EXPANSION_IPV4,
 447                          MLX5_EXPANSION_IPV6),
 448                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 449                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 450                         ETH_RSS_NONFRAG_IPV4_OTHER,
 451         },
 452         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 453                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 454                                                   MLX5_EXPANSION_VXLAN_GPE),
 455                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 456                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 457         },
 458         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 459                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 460                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 461         },
 462         [MLX5_EXPANSION_OUTER_IPV6] = {
 463                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 464                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 465                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 466                          MLX5_EXPANSION_IPV4,
 467                          MLX5_EXPANSION_IPV6),
 468                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 469                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 470                         ETH_RSS_NONFRAG_IPV6_OTHER,
 471         },
 472         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 473                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 474                                                   MLX5_EXPANSION_VXLAN_GPE),
 475                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 476                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 477         },
 478         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 479                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 480                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 481         },
 482         [MLX5_EXPANSION_VXLAN] = {
 483                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 484                                                   MLX5_EXPANSION_IPV4,
 485                                                   MLX5_EXPANSION_IPV6),
 486                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 487         },
 488         [MLX5_EXPANSION_VXLAN_GPE] = {
 489                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 490                                                   MLX5_EXPANSION_IPV4,
 491                                                   MLX5_EXPANSION_IPV6),
 492                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 493         },
 494         [MLX5_EXPANSION_GRE] = {
 495                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 496                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 497         },
 498         [MLX5_EXPANSION_MPLS] = {
 499                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 500                                                   MLX5_EXPANSION_IPV6),
 501                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 502         },
 503         [MLX5_EXPANSION_ETH] = {
 504                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 505                                                   MLX5_EXPANSION_IPV6),
 506                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 507         },
 508         [MLX5_EXPANSION_ETH_VLAN] = {
 509                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 510                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 511         },
 512         [MLX5_EXPANSION_VLAN] = {
 513                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 514                                                   MLX5_EXPANSION_IPV6),
 515                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 516         },
 517         [MLX5_EXPANSION_IPV4] = {
 518                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 519                                                   MLX5_EXPANSION_IPV4_TCP),
 520                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 521                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 522                         ETH_RSS_NONFRAG_IPV4_OTHER,
 523         },
 524         [MLX5_EXPANSION_IPV4_UDP] = {
 525                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 526                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 527         },
 528         [MLX5_EXPANSION_IPV4_TCP] = {
 529                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 530                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 531         },
 532         [MLX5_EXPANSION_IPV6] = {
 533                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 534                                                   MLX5_EXPANSION_IPV6_TCP),
 535                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 536                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 537                         ETH_RSS_NONFRAG_IPV6_OTHER,
 538         },
 539         [MLX5_EXPANSION_IPV6_UDP] = {
 540                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 541                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 542         },
 543         [MLX5_EXPANSION_IPV6_TCP] = {
 544                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 545                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 546         },
 547 };
 548
 549 static const struct rte_flow_ops mlx5_flow_ops = {
 550         .validate = mlx5_flow_validate,
 551         .create = mlx5_flow_create,
 552         .destroy = mlx5_flow_destroy,
 553         .flush = mlx5_flow_flush,
 554         .isolate = mlx5_flow_isolate,
 555         .query = mlx5_flow_query,
 556         .dev_dump = mlx5_flow_dev_dump,
 557         .get_aged_flows = mlx5_flow_get_aged_flows,
 558 };
 559
 560 /* Convert FDIR request to Generic flow. */
 561 struct mlx5_fdir {
 562         struct rte_flow_attr attr;
 563         struct rte_flow_item items[4];
 564         struct rte_flow_item_eth l2;
 565         struct rte_flow_item_eth l2_mask;
 566         union {
 567                 struct rte_flow_item_ipv4 ipv4;
 568                 struct rte_flow_item_ipv6 ipv6;
 569         } l3;
 570         union {
 571                 struct rte_flow_item_ipv4 ipv4;
 572                 struct rte_flow_item_ipv6 ipv6;
 573         } l3_mask;
 574         union {
 575                 struct rte_flow_item_udp udp;
 576                 struct rte_flow_item_tcp tcp;
 577         } l4;
 578         union {
 579                 struct rte_flow_item_udp udp;
 580                 struct rte_flow_item_tcp tcp;
 581         } l4_mask;
 582         struct rte_flow_action actions[2];
 583         struct rte_flow_action_queue queue;
 584 };
 585
 586 /* Tunnel information. */
 587 struct mlx5_flow_tunnel_info {
 588         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 589         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 590 };
 591
 592 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 593         {
 594                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 595                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 596         },
 597         {
 598                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 599                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 600         },
 601         {
 602                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 603                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 604         },
 605         {
 606                 .tunnel = MLX5_FLOW_LAYER_GRE,
 607                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 608         },
 609         {
 610                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 611                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 612         },
 613         {
 614                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 615                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 616         },
 617         {
 618                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 619                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 620         },
 621         {
 622                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 623                 .ptype = RTE_PTYPE_TUNNEL_IP,
 624         },
 625         {
 626                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 627                 .ptype = RTE_PTYPE_TUNNEL_IP,
 628         },
 629         {
 630                 .tunnel = MLX5_FLOW_LAYER_GTP,
 631                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 632         },
 633 };
 634
 635 /**
 636  * Translate tag ID to register.
 637  *
 638  * @param[in] dev
 639  *   Pointer to the Ethernet device structure.
 640  * @param[in] feature
 641  *   The feature that request the register.
 642  * @param[in] id
 643  *   The request register ID.
 644  * @param[out] error
 645  *   Error description in case of any.
 646  *
 647  * @return
 648  *   The request register on success, a negative errno
 649  *   value otherwise and rte_errno is set.
 650  */
 651 int
 652 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 653                      enum mlx5_feature_name feature,
 654                      uint32_t id,
 655                      struct rte_flow_error *error)
 656 {
 657         struct mlx5_priv *priv = dev->data->dev_private;
 658         struct mlx5_dev_config *config = &priv->config;
 659         enum modify_reg start_reg;
 660         bool skip_mtr_reg = false;
 661
 662         switch (feature) {
 663         case MLX5_HAIRPIN_RX:
 664                 return REG_B;
 665         case MLX5_HAIRPIN_TX:
 666                 return REG_A;
 667         case MLX5_METADATA_RX:
 668                 switch (config->dv_xmeta_en) {
 669                 case MLX5_XMETA_MODE_LEGACY:
 670                         return REG_B;
 671                 case MLX5_XMETA_MODE_META16:
 672                         return REG_C_0;
 673                 case MLX5_XMETA_MODE_META32:
 674                         return REG_C_1;
 675                 }
 676                 break;
 677         case MLX5_METADATA_TX:
 678                 return REG_A;
 679         case MLX5_METADATA_FDB:
 680                 switch (config->dv_xmeta_en) {
 681                 case MLX5_XMETA_MODE_LEGACY:
 682                         return REG_NON;
 683                 case MLX5_XMETA_MODE_META16:
 684                         return REG_C_0;
 685                 case MLX5_XMETA_MODE_META32:
 686                         return REG_C_1;
 687                 }
 688                 break;
 689         case MLX5_FLOW_MARK:
 690                 switch (config->dv_xmeta_en) {
 691                 case MLX5_XMETA_MODE_LEGACY:
 692                         return REG_NON;
 693                 case MLX5_XMETA_MODE_META16:
 694                         return REG_C_1;
 695                 case MLX5_XMETA_MODE_META32:
 696                         return REG_C_0;
 697                 }
 698                 break;
 699         case MLX5_MTR_SFX:
 700                 /*
 701                  * If meter color and flow match share one register, flow match
 702                  * should use the meter color register for match.
 703                  */
 704                 if (priv->mtr_reg_share)
 705                         return priv->mtr_color_reg;
 706                 else
 707                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 708                                REG_C_3;
 709         case MLX5_MTR_COLOR:
 710                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 711                 return priv->mtr_color_reg;
 712         case MLX5_COPY_MARK:
 713                 /*
 714                  * Metadata COPY_MARK register using is in meter suffix sub
 715                  * flow while with meter. It's safe to share the same register.
 716                  */
 717                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 718         case MLX5_APP_TAG:
 719                 /*
 720                  * If meter is enable, it will engage the register for color
 721                  * match and flow match. If meter color match is not using the
 722                  * REG_C_2, need to skip the REG_C_x be used by meter color
 723                  * match.
 724                  * If meter is disable, free to use all available registers.
 725                  */
 726                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 727                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 728                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 729                 if (id > (REG_C_7 - start_reg))
 730                         return rte_flow_error_set(error, EINVAL,
 731                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 732                                                   NULL, "invalid tag id");
 733                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 734                         return rte_flow_error_set(error, ENOTSUP,
 735                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 736                                                   NULL, "unsupported tag id");
 737                 /*
 738                  * This case means meter is using the REG_C_x great than 2.
 739                  * Take care not to conflict with meter color REG_C_x.
 740                  * If the available index REG_C_y >= REG_C_x, skip the
 741                  * color register.
 742                  */
 743                 if (skip_mtr_reg && config->flow_mreg_c
 744                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 745                         if (id >= (REG_C_7 - start_reg))
 746                                 return rte_flow_error_set(error, EINVAL,
 747                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 748                                                         NULL, "invalid tag id");
 749                         if (config->flow_mreg_c
 750                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 751                                 return config->flow_mreg_c
 752                                                [id + 1 + start_reg - REG_C_0];
 753                         return rte_flow_error_set(error, ENOTSUP,
 754                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 755                                                   NULL, "unsupported tag id");
 756                 }
 757                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 758         }
 759         MLX5_ASSERT(false);
 760         return rte_flow_error_set(error, EINVAL,
 761                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 762                                   NULL, "invalid feature name");
 763 }
 764
 765 /**
 766  * Check extensive flow metadata register support.
 767  *
 768  * @param dev
 769  *   Pointer to rte_eth_dev structure.
 770  *
 771  * @return
 772  *   True if device supports extensive flow metadata register, otherwise false.
 773  */
 774 bool
 775 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 776 {
 777         struct mlx5_priv *priv = dev->data->dev_private;
 778         struct mlx5_dev_config *config = &priv->config;
 779
 780         /*
 781          * Having available reg_c can be regarded inclusively as supporting
 782          * extensive flow metadata register, which could mean,
 783          * - metadata register copy action by modify header.
 784          * - 16 modify header actions is supported.
 785          * - reg_c's are preserved across different domain (FDB and NIC) on
 786          *   packet loopback by flow lookup miss.
 787          */
 788         return config->flow_mreg_c[2] != REG_NON;
 789 }
 790
 791 /**
 792  * Verify the @p item specifications (spec, last, mask) are compatible with the
 793  * NIC capabilities.
 794  *
 795  * @param[in] item
 796  *   Item specification.
 797  * @param[in] mask
 798  *   @p item->mask or flow default bit-masks.
 799  * @param[in] nic_mask
 800  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 801  * @param[in] size
 802  *   Bit-masks size in bytes.
 803  * @param[out] error
 804  *   Pointer to error structure.
 805  *
 806  * @return
 807  *   0 on success, a negative errno value otherwise and rte_errno is set.
 808  */
 809 int
 810 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 811                           const uint8_t *mask,
 812                           const uint8_t *nic_mask,
 813                           unsigned int size,
 814                           struct rte_flow_error *error)
 815 {
 816         unsigned int i;
 817
 818         MLX5_ASSERT(nic_mask);
 819         for (i = 0; i < size; ++i)
 820                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 821                         return rte_flow_error_set(error, ENOTSUP,
 822                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 823                                                   item,
 824                                                   "mask enables non supported"
 825                                                   " bits");
 826         if (!item->spec && (item->mask || item->last))
 827                 return rte_flow_error_set(error, EINVAL,
 828                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 829                                           "mask/last without a spec is not"
 830                                           " supported");
 831         if (item->spec && item->last) {
 832                 uint8_t spec[size];
 833                 uint8_t last[size];
 834                 unsigned int i;
 835                 int ret;
 836
 837                 for (i = 0; i < size; ++i) {
 838                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 839                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 840                 }
 841                 ret = memcmp(spec, last, size);
 842                 if (ret != 0)
 843                         return rte_flow_error_set(error, EINVAL,
 844                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 845                                                   item,
 846                                                   "range is not valid");
 847         }
 848         return 0;
 849 }
 850
 851 /**
 852  * Adjust the hash fields according to the @p flow information.
 853  *
 854  * @param[in] dev_flow.
 855  *   Pointer to the mlx5_flow.
 856  * @param[in] tunnel
 857  *   1 when the hash field is for a tunnel item.
 858  * @param[in] layer_types
 859  *   ETH_RSS_* types.
 860  * @param[in] hash_fields
 861  *   Item hash fields.
 862  *
 863  * @return
 864  *   The hash fields that should be used.
 865  */
 866 uint64_t
 867 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
 868                             int tunnel __rte_unused, uint64_t layer_types,
 869                             uint64_t hash_fields)
 870 {
 871 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 872         int rss_request_inner = rss_desc->level >= 2;
 873
 874         /* Check RSS hash level for tunnel. */
 875         if (tunnel && rss_request_inner)
 876                 hash_fields |= IBV_RX_HASH_INNER;
 877         else if (tunnel || rss_request_inner)
 878                 return 0;
 879 #endif
 880         /* Check if requested layer matches RSS hash fields. */
 881         if (!(rss_desc->types & layer_types))
 882                 return 0;
 883         return hash_fields;
 884 }
 885
 886 /**
 887  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 888  * if several tunnel rules are used on this queue, the tunnel ptype will be
 889  * cleared.
 890  *
 891  * @param rxq_ctrl
 892  *   Rx queue to update.
 893  */
 894 static void
 895 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 896 {
 897         unsigned int i;
 898         uint32_t tunnel_ptype = 0;
 899
 900         /* Look up for the ptype to use. */
 901         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 902                 if (!rxq_ctrl->flow_tunnels_n[i])
 903                         continue;
 904                 if (!tunnel_ptype) {
 905                         tunnel_ptype = tunnels_info[i].ptype;
 906                 } else {
 907                         tunnel_ptype = 0;
 908                         break;
 909                 }
 910         }
 911         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 912 }
 913
 914 /**
 915  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 916  * flow.
 917  *
 918  * @param[in] dev
 919  *   Pointer to the Ethernet device structure.
 920  * @param[in] dev_handle
 921  *   Pointer to device flow handle structure.
 922  */
 923 static void
 924 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
 925                        struct mlx5_flow_handle *dev_handle)
 926 {
 927         struct mlx5_priv *priv = dev->data->dev_private;
 928         const int mark = dev_handle->mark;
 929         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
 930         struct mlx5_hrxq *hrxq;
 931         unsigned int i;
 932
 933         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
 934                 return;
 935         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
 936                               dev_handle->rix_hrxq);
 937         if (!hrxq)
 938                 return;
 939         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
 940                 int idx = hrxq->ind_table->queues[i];
 941                 struct mlx5_rxq_ctrl *rxq_ctrl =
 942                         container_of((*priv->rxqs)[idx],
 943                                      struct mlx5_rxq_ctrl, rxq);
 944
 945                 /*
 946                  * To support metadata register copy on Tx loopback,
 947                  * this must be always enabled (metadata may arive
 948                  * from other port - not from local flows only.
 949                  */
 950                 if (priv->config.dv_flow_en &&
 951                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
 952                     mlx5_flow_ext_mreg_supported(dev)) {
 953                         rxq_ctrl->rxq.mark = 1;
 954                         rxq_ctrl->flow_mark_n = 1;
 955                 } else if (mark) {
 956                         rxq_ctrl->rxq.mark = 1;
 957                         rxq_ctrl->flow_mark_n++;
 958                 }
 959                 if (tunnel) {
 960                         unsigned int j;
 961
 962                         /* Increase the counter matching the flow. */
 963                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 964                                 if ((tunnels_info[j].tunnel &
 965                                      dev_handle->layers) ==
 966                                     tunnels_info[j].tunnel) {
 967                                         rxq_ctrl->flow_tunnels_n[j]++;
 968                                         break;
 969                                 }
 970                         }
 971                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 972                 }
 973         }
 974 }
 975
 976 /**
 977  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
 978  *
 979  * @param[in] dev
 980  *   Pointer to the Ethernet device structure.
 981  * @param[in] flow
 982  *   Pointer to flow structure.
 983  */
 984 static void
 985 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 986 {
 987         struct mlx5_priv *priv = dev->data->dev_private;
 988         uint32_t handle_idx;
 989         struct mlx5_flow_handle *dev_handle;
 990
 991         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
 992                        handle_idx, dev_handle, next)
 993                 flow_drv_rxq_flags_set(dev, dev_handle);
 994 }
 995
 996 /**
 997  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 998  * device flow if no other flow uses it with the same kind of request.
 999  *
1000  * @param dev
1001  *   Pointer to Ethernet device.
1002  * @param[in] dev_handle
1003  *   Pointer to the device flow handle structure.
1004  */
1005 static void
1006 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1007                         struct mlx5_flow_handle *dev_handle)
1008 {
1009         struct mlx5_priv *priv = dev->data->dev_private;
1010         const int mark = dev_handle->mark;
1011         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1012         struct mlx5_hrxq *hrxq;
1013         unsigned int i;
1014
1015         if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
1016                 return;
1017         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1018                               dev_handle->rix_hrxq);
1019         if (!hrxq)
1020                 return;
1021         MLX5_ASSERT(dev->data->dev_started);
1022         for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
1023                 int idx = hrxq->ind_table->queues[i];
1024                 struct mlx5_rxq_ctrl *rxq_ctrl =
1025                         container_of((*priv->rxqs)[idx],
1026                                      struct mlx5_rxq_ctrl, rxq);
1027
1028                 if (priv->config.dv_flow_en &&
1029                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1030                     mlx5_flow_ext_mreg_supported(dev)) {
1031                         rxq_ctrl->rxq.mark = 1;
1032                         rxq_ctrl->flow_mark_n = 1;
1033                 } else if (mark) {
1034                         rxq_ctrl->flow_mark_n--;
1035                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1036                 }
1037                 if (tunnel) {
1038                         unsigned int j;
1039
1040                         /* Decrease the counter matching the flow. */
1041                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1042                                 if ((tunnels_info[j].tunnel &
1043                                      dev_handle->layers) ==
1044                                     tunnels_info[j].tunnel) {
1045                                         rxq_ctrl->flow_tunnels_n[j]--;
1046                                         break;
1047                                 }
1048                         }
1049                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1050                 }
1051         }
1052 }
1053
1054 /**
1055  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1056  * @p flow if no other flow uses it with the same kind of request.
1057  *
1058  * @param dev
1059  *   Pointer to Ethernet device.
1060  * @param[in] flow
1061  *   Pointer to the flow.
1062  */
1063 static void
1064 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1065 {
1066         struct mlx5_priv *priv = dev->data->dev_private;
1067         uint32_t handle_idx;
1068         struct mlx5_flow_handle *dev_handle;
1069
1070         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1071                        handle_idx, dev_handle, next)
1072                 flow_drv_rxq_flags_trim(dev, dev_handle);
1073 }
1074
1075 /**
1076  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1077  *
1078  * @param dev
1079  *   Pointer to Ethernet device.
1080  */
1081 static void
1082 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1083 {
1084         struct mlx5_priv *priv = dev->data->dev_private;
1085         unsigned int i;
1086
1087         for (i = 0; i != priv->rxqs_n; ++i) {
1088                 struct mlx5_rxq_ctrl *rxq_ctrl;
1089                 unsigned int j;
1090
1091                 if (!(*priv->rxqs)[i])
1092                         continue;
1093                 rxq_ctrl = container_of((*priv->rxqs)[i],
1094                                         struct mlx5_rxq_ctrl, rxq);
1095                 rxq_ctrl->flow_mark_n = 0;
1096                 rxq_ctrl->rxq.mark = 0;
1097                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1098                         rxq_ctrl->flow_tunnels_n[j] = 0;
1099                 rxq_ctrl->rxq.tunnel = 0;
1100         }
1101 }
1102
1103 /**
1104  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1105  *
1106  * @param[in] dev
1107  *   Pointer to the Ethernet device structure.
1108  */
1109 void
1110 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1111 {
1112         struct mlx5_priv *priv = dev->data->dev_private;
1113         struct mlx5_rxq_data *data;
1114         unsigned int i;
1115
1116         for (i = 0; i != priv->rxqs_n; ++i) {
1117                 if (!(*priv->rxqs)[i])
1118                         continue;
1119                 data = (*priv->rxqs)[i];
1120                 if (!rte_flow_dynf_metadata_avail()) {
1121                         data->dynf_meta = 0;
1122                         data->flow_meta_mask = 0;
1123                         data->flow_meta_offset = -1;
1124                 } else {
1125                         data->dynf_meta = 1;
1126                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1127                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1128                 }
1129         }
1130 }
1131
1132 /*
1133  * return a pointer to the desired action in the list of actions.
1134  *
1135  * @param[in] actions
1136  *   The list of actions to search the action in.
1137  * @param[in] action
1138  *   The action to find.
1139  *
1140  * @return
1141  *   Pointer to the action in the list, if found. NULL otherwise.
1142  */
1143 const struct rte_flow_action *
1144 mlx5_flow_find_action(const struct rte_flow_action *actions,
1145                       enum rte_flow_action_type action)
1146 {
1147         if (actions == NULL)
1148                 return NULL;
1149         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1150                 if (actions->type == action)
1151                         return actions;
1152         return NULL;
1153 }
1154
1155 /*
1156  * Validate the flag action.
1157  *
1158  * @param[in] action_flags
1159  *   Bit-fields that holds the actions detected until now.
1160  * @param[in] attr
1161  *   Attributes of flow that includes this action.
1162  * @param[out] error
1163  *   Pointer to error structure.
1164  *
1165  * @return
1166  *   0 on success, a negative errno value otherwise and rte_errno is set.
1167  */
1168 int
1169 mlx5_flow_validate_action_flag(uint64_t action_flags,
1170                                const struct rte_flow_attr *attr,
1171                                struct rte_flow_error *error)
1172 {
1173         if (action_flags & MLX5_FLOW_ACTION_MARK)
1174                 return rte_flow_error_set(error, EINVAL,
1175                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1176                                           "can't mark and flag in same flow");
1177         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1178                 return rte_flow_error_set(error, EINVAL,
1179                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1180                                           "can't have 2 flag"
1181                                           " actions in same flow");
1182         if (attr->egress)
1183                 return rte_flow_error_set(error, ENOTSUP,
1184                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1185                                           "flag action not supported for "
1186                                           "egress");
1187         return 0;
1188 }
1189
1190 /*
1191  * Validate the mark action.
1192  *
1193  * @param[in] action
1194  *   Pointer to the queue action.
1195  * @param[in] action_flags
1196  *   Bit-fields that holds the actions detected until now.
1197  * @param[in] attr
1198  *   Attributes of flow that includes this action.
1199  * @param[out] error
1200  *   Pointer to error structure.
1201  *
1202  * @return
1203  *   0 on success, a negative errno value otherwise and rte_errno is set.
1204  */
1205 int
1206 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1207                                uint64_t action_flags,
1208                                const struct rte_flow_attr *attr,
1209                                struct rte_flow_error *error)
1210 {
1211         const struct rte_flow_action_mark *mark = action->conf;
1212
1213         if (!mark)
1214                 return rte_flow_error_set(error, EINVAL,
1215                                           RTE_FLOW_ERROR_TYPE_ACTION,
1216                                           action,
1217                                           "configuration cannot be null");
1218         if (mark->id >= MLX5_FLOW_MARK_MAX)
1219                 return rte_flow_error_set(error, EINVAL,
1220                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1221                                           &mark->id,
1222                                           "mark id must in 0 <= id < "
1223                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1224         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1225                 return rte_flow_error_set(error, EINVAL,
1226                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1227                                           "can't flag and mark in same flow");
1228         if (action_flags & MLX5_FLOW_ACTION_MARK)
1229                 return rte_flow_error_set(error, EINVAL,
1230                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1231                                           "can't have 2 mark actions in same"
1232                                           " flow");
1233         if (attr->egress)
1234                 return rte_flow_error_set(error, ENOTSUP,
1235                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1236                                           "mark action not supported for "
1237                                           "egress");
1238         return 0;
1239 }
1240
1241 /*
1242  * Validate the drop action.
1243  *
1244  * @param[in] action_flags
1245  *   Bit-fields that holds the actions detected until now.
1246  * @param[in] attr
1247  *   Attributes of flow that includes this action.
1248  * @param[out] error
1249  *   Pointer to error structure.
1250  *
1251  * @return
1252  *   0 on success, a negative errno value otherwise and rte_errno is set.
1253  */
1254 int
1255 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1256                                const struct rte_flow_attr *attr,
1257                                struct rte_flow_error *error)
1258 {
1259         if (attr->egress)
1260                 return rte_flow_error_set(error, ENOTSUP,
1261                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1262                                           "drop action not supported for "
1263                                           "egress");
1264         return 0;
1265 }
1266
1267 /*
1268  * Validate the queue action.
1269  *
1270  * @param[in] action
1271  *   Pointer to the queue action.
1272  * @param[in] action_flags
1273  *   Bit-fields that holds the actions detected until now.
1274  * @param[in] dev
1275  *   Pointer to the Ethernet device structure.
1276  * @param[in] attr
1277  *   Attributes of flow that includes this action.
1278  * @param[out] error
1279  *   Pointer to error structure.
1280  *
1281  * @return
1282  *   0 on success, a negative errno value otherwise and rte_errno is set.
1283  */
1284 int
1285 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1286                                 uint64_t action_flags,
1287                                 struct rte_eth_dev *dev,
1288                                 const struct rte_flow_attr *attr,
1289                                 struct rte_flow_error *error)
1290 {
1291         struct mlx5_priv *priv = dev->data->dev_private;
1292         const struct rte_flow_action_queue *queue = action->conf;
1293
1294         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1295                 return rte_flow_error_set(error, EINVAL,
1296                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1297                                           "can't have 2 fate actions in"
1298                                           " same flow");
1299         if (!priv->rxqs_n)
1300                 return rte_flow_error_set(error, EINVAL,
1301                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1302                                           NULL, "No Rx queues configured");
1303         if (queue->index >= priv->rxqs_n)
1304                 return rte_flow_error_set(error, EINVAL,
1305                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1306                                           &queue->index,
1307                                           "queue index out of range");
1308         if (!(*priv->rxqs)[queue->index])
1309                 return rte_flow_error_set(error, EINVAL,
1310                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1311                                           &queue->index,
1312                                           "queue is not configured");
1313         if (attr->egress)
1314                 return rte_flow_error_set(error, ENOTSUP,
1315                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1316                                           "queue action not supported for "
1317                                           "egress");
1318         return 0;
1319 }
1320
1321 /*
1322  * Validate the rss action.
1323  *
1324  * @param[in] action
1325  *   Pointer to the queue action.
1326  * @param[in] action_flags
1327  *   Bit-fields that holds the actions detected until now.
1328  * @param[in] dev
1329  *   Pointer to the Ethernet device structure.
1330  * @param[in] attr
1331  *   Attributes of flow that includes this action.
1332  * @param[in] item_flags
1333  *   Items that were detected.
1334  * @param[out] error
1335  *   Pointer to error structure.
1336  *
1337  * @return
1338  *   0 on success, a negative errno value otherwise and rte_errno is set.
1339  */
1340 int
1341 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1342                               uint64_t action_flags,
1343                               struct rte_eth_dev *dev,
1344                               const struct rte_flow_attr *attr,
1345                               uint64_t item_flags,
1346                               struct rte_flow_error *error)
1347 {
1348         struct mlx5_priv *priv = dev->data->dev_private;
1349         const struct rte_flow_action_rss *rss = action->conf;
1350         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1351         unsigned int i;
1352
1353         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1354                 return rte_flow_error_set(error, EINVAL,
1355                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1356                                           "can't have 2 fate actions"
1357                                           " in same flow");
1358         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1359             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1360                 return rte_flow_error_set(error, ENOTSUP,
1361                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1362                                           &rss->func,
1363                                           "RSS hash function not supported");
1364 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1365         if (rss->level > 2)
1366 #else
1367         if (rss->level > 1)
1368 #endif
1369                 return rte_flow_error_set(error, ENOTSUP,
1370                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1371                                           &rss->level,
1372                                           "tunnel RSS is not supported");
1373         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1374         if (rss->key_len == 0 && rss->key != NULL)
1375                 return rte_flow_error_set(error, ENOTSUP,
1376                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1377                                           &rss->key_len,
1378                                           "RSS hash key length 0");
1379         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1380                 return rte_flow_error_set(error, ENOTSUP,
1381                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1382                                           &rss->key_len,
1383                                           "RSS hash key too small");
1384         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1385                 return rte_flow_error_set(error, ENOTSUP,
1386                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1387                                           &rss->key_len,
1388                                           "RSS hash key too large");
1389         if (rss->queue_num > priv->config.ind_table_max_size)
1390                 return rte_flow_error_set(error, ENOTSUP,
1391                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1392                                           &rss->queue_num,
1393                                           "number of queues too large");
1394         if (rss->types & MLX5_RSS_HF_MASK)
1395                 return rte_flow_error_set(error, ENOTSUP,
1396                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1397                                           &rss->types,
1398                                           "some RSS protocols are not"
1399                                           " supported");
1400         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1401             !(rss->types & ETH_RSS_IP))
1402                 return rte_flow_error_set(error, EINVAL,
1403                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1404                                           "L3 partial RSS requested but L3 RSS"
1405                                           " type not specified");
1406         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1407             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1408                 return rte_flow_error_set(error, EINVAL,
1409                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1410                                           "L4 partial RSS requested but L4 RSS"
1411                                           " type not specified");
1412         if (!priv->rxqs_n)
1413                 return rte_flow_error_set(error, EINVAL,
1414                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1415                                           NULL, "No Rx queues configured");
1416         if (!rss->queue_num)
1417                 return rte_flow_error_set(error, EINVAL,
1418                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1419                                           NULL, "No queues configured");
1420         for (i = 0; i != rss->queue_num; ++i) {
1421                 if (rss->queue[i] >= priv->rxqs_n)
1422                         return rte_flow_error_set
1423                                 (error, EINVAL,
1424                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1425                                  &rss->queue[i], "queue index out of range");
1426                 if (!(*priv->rxqs)[rss->queue[i]])
1427                         return rte_flow_error_set
1428                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1429                                  &rss->queue[i], "queue is not configured");
1430         }
1431         if (attr->egress)
1432                 return rte_flow_error_set(error, ENOTSUP,
1433                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1434                                           "rss action not supported for "
1435                                           "egress");
1436         if (rss->level > 1 && !tunnel)
1437                 return rte_flow_error_set(error, EINVAL,
1438                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1439                                           "inner RSS is not supported for "
1440                                           "non-tunnel flows");
1441         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1442             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1443                 return rte_flow_error_set(error, EINVAL,
1444                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1445                                           "RSS on eCPRI is not supported now");
1446         }
1447         return 0;
1448 }
1449
1450 /*
1451  * Validate the default miss action.
1452  *
1453  * @param[in] action_flags
1454  *   Bit-fields that holds the actions detected until now.
1455  * @param[out] error
1456  *   Pointer to error structure.
1457  *
1458  * @return
1459  *   0 on success, a negative errno value otherwise and rte_errno is set.
1460  */
1461 int
1462 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1463                                 const struct rte_flow_attr *attr,
1464                                 struct rte_flow_error *error)
1465 {
1466         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1469                                           "can't have 2 fate actions in"
1470                                           " same flow");
1471         if (attr->egress)
1472                 return rte_flow_error_set(error, ENOTSUP,
1473                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1474                                           "default miss action not supported "
1475                                           "for egress");
1476         if (attr->group)
1477                 return rte_flow_error_set(error, ENOTSUP,
1478                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1479                                           "only group 0 is supported");
1480         if (attr->transfer)
1481                 return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1483                                           NULL, "transfer is not supported");
1484         return 0;
1485 }
1486
1487 /*
1488  * Validate the count action.
1489  *
1490  * @param[in] dev
1491  *   Pointer to the Ethernet device structure.
1492  * @param[in] attr
1493  *   Attributes of flow that includes this action.
1494  * @param[out] error
1495  *   Pointer to error structure.
1496  *
1497  * @return
1498  *   0 on success, a negative errno value otherwise and rte_errno is set.
1499  */
1500 int
1501 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1502                                 const struct rte_flow_attr *attr,
1503                                 struct rte_flow_error *error)
1504 {
1505         if (attr->egress)
1506                 return rte_flow_error_set(error, ENOTSUP,
1507                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1508                                           "count action not supported for "
1509                                           "egress");
1510         return 0;
1511 }
1512
1513 /**
1514  * Verify the @p attributes will be correctly understood by the NIC and store
1515  * them in the @p flow if everything is correct.
1516  *
1517  * @param[in] dev
1518  *   Pointer to the Ethernet device structure.
1519  * @param[in] attributes
1520  *   Pointer to flow attributes
1521  * @param[out] error
1522  *   Pointer to error structure.
1523  *
1524  * @return
1525  *   0 on success, a negative errno value otherwise and rte_errno is set.
1526  */
1527 int
1528 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1529                               const struct rte_flow_attr *attributes,
1530                               struct rte_flow_error *error)
1531 {
1532         struct mlx5_priv *priv = dev->data->dev_private;
1533         uint32_t priority_max = priv->config.flow_prio - 1;
1534
1535         if (attributes->group)
1536                 return rte_flow_error_set(error, ENOTSUP,
1537                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1538                                           NULL, "groups is not supported");
1539         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1540             attributes->priority >= priority_max)
1541                 return rte_flow_error_set(error, ENOTSUP,
1542                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1543                                           NULL, "priority out of range");
1544         if (attributes->egress)
1545                 return rte_flow_error_set(error, ENOTSUP,
1546                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1547                                           "egress is not supported");
1548         if (attributes->transfer && !priv->config.dv_esw_en)
1549                 return rte_flow_error_set(error, ENOTSUP,
1550                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1551                                           NULL, "transfer is not supported");
1552         if (!attributes->ingress)
1553                 return rte_flow_error_set(error, EINVAL,
1554                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1555                                           NULL,
1556                                           "ingress attribute is mandatory");
1557         return 0;
1558 }
1559
1560 /**
1561  * Validate ICMP6 item.
1562  *
1563  * @param[in] item
1564  *   Item specification.
1565  * @param[in] item_flags
1566  *   Bit-fields that holds the items detected until now.
1567  * @param[out] error
1568  *   Pointer to error structure.
1569  *
1570  * @return
1571  *   0 on success, a negative errno value otherwise and rte_errno is set.
1572  */
1573 int
1574 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1575                                uint64_t item_flags,
1576                                uint8_t target_protocol,
1577                                struct rte_flow_error *error)
1578 {
1579         const struct rte_flow_item_icmp6 *mask = item->mask;
1580         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1581         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1582                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1583         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1584                                       MLX5_FLOW_LAYER_OUTER_L4;
1585         int ret;
1586
1587         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1588                 return rte_flow_error_set(error, EINVAL,
1589                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1590                                           "protocol filtering not compatible"
1591                                           " with ICMP6 layer");
1592         if (!(item_flags & l3m))
1593                 return rte_flow_error_set(error, EINVAL,
1594                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1595                                           "IPv6 is mandatory to filter on"
1596                                           " ICMP6");
1597         if (item_flags & l4m)
1598                 return rte_flow_error_set(error, EINVAL,
1599                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1600                                           "multiple L4 layers not supported");
1601         if (!mask)
1602                 mask = &rte_flow_item_icmp6_mask;
1603         ret = mlx5_flow_item_acceptable
1604                 (item, (const uint8_t *)mask,
1605                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1606                  sizeof(struct rte_flow_item_icmp6), error);
1607         if (ret < 0)
1608                 return ret;
1609         return 0;
1610 }
1611
1612 /**
1613  * Validate ICMP item.
1614  *
1615  * @param[in] item
1616  *   Item specification.
1617  * @param[in] item_flags
1618  *   Bit-fields that holds the items detected until now.
1619  * @param[out] error
1620  *   Pointer to error structure.
1621  *
1622  * @return
1623  *   0 on success, a negative errno value otherwise and rte_errno is set.
1624  */
1625 int
1626 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1627                              uint64_t item_flags,
1628                              uint8_t target_protocol,
1629                              struct rte_flow_error *error)
1630 {
1631         const struct rte_flow_item_icmp *mask = item->mask;
1632         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1633         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1634                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1635         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1636                                       MLX5_FLOW_LAYER_OUTER_L4;
1637         int ret;
1638
1639         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1640                 return rte_flow_error_set(error, EINVAL,
1641                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1642                                           "protocol filtering not compatible"
1643                                           " with ICMP layer");
1644         if (!(item_flags & l3m))
1645                 return rte_flow_error_set(error, EINVAL,
1646                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1647                                           "IPv4 is mandatory to filter"
1648                                           " on ICMP");
1649         if (item_flags & l4m)
1650                 return rte_flow_error_set(error, EINVAL,
1651                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1652                                           "multiple L4 layers not supported");
1653         if (!mask)
1654                 mask = &rte_flow_item_icmp_mask;
1655         ret = mlx5_flow_item_acceptable
1656                 (item, (const uint8_t *)mask,
1657                  (const uint8_t *)&rte_flow_item_icmp_mask,
1658                  sizeof(struct rte_flow_item_icmp), error);
1659         if (ret < 0)
1660                 return ret;
1661         return 0;
1662 }
1663
1664 /**
1665  * Validate Ethernet item.
1666  *
1667  * @param[in] item
1668  *   Item specification.
1669  * @param[in] item_flags
1670  *   Bit-fields that holds the items detected until now.
1671  * @param[out] error
1672  *   Pointer to error structure.
1673  *
1674  * @return
1675  *   0 on success, a negative errno value otherwise and rte_errno is set.
1676  */
1677 int
1678 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1679                             uint64_t item_flags,
1680                             struct rte_flow_error *error)
1681 {
1682         const struct rte_flow_item_eth *mask = item->mask;
1683         const struct rte_flow_item_eth nic_mask = {
1684                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1685                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1686                 .type = RTE_BE16(0xffff),
1687         };
1688         int ret;
1689         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1690         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1691                                        MLX5_FLOW_LAYER_OUTER_L2;
1692
1693         if (item_flags & ethm)
1694                 return rte_flow_error_set(error, ENOTSUP,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "multiple L2 layers not supported");
1697         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1698             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1699                 return rte_flow_error_set(error, EINVAL,
1700                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1701                                           "L2 layer should not follow "
1702                                           "L3 layers");
1703         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1704             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1705                 return rte_flow_error_set(error, EINVAL,
1706                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1707                                           "L2 layer should not follow VLAN");
1708         if (!mask)
1709                 mask = &rte_flow_item_eth_mask;
1710         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1711                                         (const uint8_t *)&nic_mask,
1712                                         sizeof(struct rte_flow_item_eth),
1713                                         error);
1714         return ret;
1715 }
1716
1717 /**
1718  * Validate VLAN item.
1719  *
1720  * @param[in] item
1721  *   Item specification.
1722  * @param[in] item_flags
1723  *   Bit-fields that holds the items detected until now.
1724  * @param[in] dev
1725  *   Ethernet device flow is being created on.
1726  * @param[out] error
1727  *   Pointer to error structure.
1728  *
1729  * @return
1730  *   0 on success, a negative errno value otherwise and rte_errno is set.
1731  */
1732 int
1733 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1734                              uint64_t item_flags,
1735                              struct rte_eth_dev *dev,
1736                              struct rte_flow_error *error)
1737 {
1738         const struct rte_flow_item_vlan *spec = item->spec;
1739         const struct rte_flow_item_vlan *mask = item->mask;
1740         const struct rte_flow_item_vlan nic_mask = {
1741                 .tci = RTE_BE16(UINT16_MAX),
1742                 .inner_type = RTE_BE16(UINT16_MAX),
1743         };
1744         uint16_t vlan_tag = 0;
1745         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1746         int ret;
1747         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1748                                         MLX5_FLOW_LAYER_INNER_L4) :
1749                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1750                                         MLX5_FLOW_LAYER_OUTER_L4);
1751         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1752                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1753
1754         if (item_flags & vlanm)
1755                 return rte_flow_error_set(error, EINVAL,
1756                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1757                                           "multiple VLAN layers not supported");
1758         else if ((item_flags & l34m) != 0)
1759                 return rte_flow_error_set(error, EINVAL,
1760                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1761                                           "VLAN cannot follow L3/L4 layer");
1762         if (!mask)
1763                 mask = &rte_flow_item_vlan_mask;
1764         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1765                                         (const uint8_t *)&nic_mask,
1766                                         sizeof(struct rte_flow_item_vlan),
1767                                         error);
1768         if (ret)
1769                 return ret;
1770         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1771                 struct mlx5_priv *priv = dev->data->dev_private;
1772
1773                 if (priv->vmwa_context) {
1774                         /*
1775                          * Non-NULL context means we have a virtual machine
1776                          * and SR-IOV enabled, we have to create VLAN interface
1777                          * to make hypervisor to setup E-Switch vport
1778                          * context correctly. We avoid creating the multiple
1779                          * VLAN interfaces, so we cannot support VLAN tag mask.
1780                          */
1781                         return rte_flow_error_set(error, EINVAL,
1782                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1783                                                   item,
1784                                                   "VLAN tag mask is not"
1785                                                   " supported in virtual"
1786                                                   " environment");
1787                 }
1788         }
1789         if (spec) {
1790                 vlan_tag = spec->tci;
1791                 vlan_tag &= mask->tci;
1792         }
1793         /*
1794          * From verbs perspective an empty VLAN is equivalent
1795          * to a packet without VLAN layer.
1796          */
1797         if (!vlan_tag)
1798                 return rte_flow_error_set(error, EINVAL,
1799                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1800                                           item->spec,
1801                                           "VLAN cannot be empty");
1802         return 0;
1803 }
1804
1805 /**
1806  * Validate IPV4 item.
1807  *
1808  * @param[in] item
1809  *   Item specification.
1810  * @param[in] item_flags
1811  *   Bit-fields that holds the items detected until now.
1812  * @param[in] last_item
1813  *   Previous validated item in the pattern items.
1814  * @param[in] ether_type
1815  *   Type in the ethernet layer header (including dot1q).
1816  * @param[in] acc_mask
1817  *   Acceptable mask, if NULL default internal default mask
1818  *   will be used to check whether item fields are supported.
1819  * @param[out] error
1820  *   Pointer to error structure.
1821  *
1822  * @return
1823  *   0 on success, a negative errno value otherwise and rte_errno is set.
1824  */
1825 int
1826 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1827                              uint64_t item_flags,
1828                              uint64_t last_item,
1829                              uint16_t ether_type,
1830                              const struct rte_flow_item_ipv4 *acc_mask,
1831                              struct rte_flow_error *error)
1832 {
1833         const struct rte_flow_item_ipv4 *mask = item->mask;
1834         const struct rte_flow_item_ipv4 *spec = item->spec;
1835         const struct rte_flow_item_ipv4 nic_mask = {
1836                 .hdr = {
1837                         .src_addr = RTE_BE32(0xffffffff),
1838                         .dst_addr = RTE_BE32(0xffffffff),
1839                         .type_of_service = 0xff,
1840                         .next_proto_id = 0xff,
1841                 },
1842         };
1843         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1844         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1845                                       MLX5_FLOW_LAYER_OUTER_L3;
1846         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1847                                       MLX5_FLOW_LAYER_OUTER_L4;
1848         int ret;
1849         uint8_t next_proto = 0xFF;
1850         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1851                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1852                                   MLX5_FLOW_LAYER_INNER_VLAN);
1853
1854         if ((last_item & l2_vlan) && ether_type &&
1855             ether_type != RTE_ETHER_TYPE_IPV4)
1856                 return rte_flow_error_set(error, EINVAL,
1857                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1858                                           "IPv4 cannot follow L2/VLAN layer "
1859                                           "which ether type is not IPv4");
1860         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1861                 if (mask && spec)
1862                         next_proto = mask->hdr.next_proto_id &
1863                                      spec->hdr.next_proto_id;
1864                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1865                         return rte_flow_error_set(error, EINVAL,
1866                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1867                                                   item,
1868                                                   "multiple tunnel "
1869                                                   "not supported");
1870         }
1871         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1872                 return rte_flow_error_set(error, EINVAL,
1873                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1874                                           "wrong tunnel type - IPv6 specified "
1875                                           "but IPv4 item provided");
1876         if (item_flags & l3m)
1877                 return rte_flow_error_set(error, ENOTSUP,
1878                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1879                                           "multiple L3 layers not supported");
1880         else if (item_flags & l4m)
1881                 return rte_flow_error_set(error, EINVAL,
1882                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1883                                           "L3 cannot follow an L4 layer.");
1884         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1885                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1886                 return rte_flow_error_set(error, EINVAL,
1887                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1888                                           "L3 cannot follow an NVGRE layer.");
1889         if (!mask)
1890                 mask = &rte_flow_item_ipv4_mask;
1891         else if (mask->hdr.next_proto_id != 0 &&
1892                  mask->hdr.next_proto_id != 0xff)
1893                 return rte_flow_error_set(error, EINVAL,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1895                                           "partial mask is not supported"
1896                                           " for protocol");
1897         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1898                                         acc_mask ? (const uint8_t *)acc_mask
1899                                                  : (const uint8_t *)&nic_mask,
1900                                         sizeof(struct rte_flow_item_ipv4),
1901                                         error);
1902         if (ret < 0)
1903                 return ret;
1904         return 0;
1905 }
1906
1907 /**
1908  * Validate IPV6 item.
1909  *
1910  * @param[in] item
1911  *   Item specification.
1912  * @param[in] item_flags
1913  *   Bit-fields that holds the items detected until now.
1914  * @param[in] last_item
1915  *   Previous validated item in the pattern items.
1916  * @param[in] ether_type
1917  *   Type in the ethernet layer header (including dot1q).
1918  * @param[in] acc_mask
1919  *   Acceptable mask, if NULL default internal default mask
1920  *   will be used to check whether item fields are supported.
1921  * @param[out] error
1922  *   Pointer to error structure.
1923  *
1924  * @return
1925  *   0 on success, a negative errno value otherwise and rte_errno is set.
1926  */
1927 int
1928 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1929                              uint64_t item_flags,
1930                              uint64_t last_item,
1931                              uint16_t ether_type,
1932                              const struct rte_flow_item_ipv6 *acc_mask,
1933                              struct rte_flow_error *error)
1934 {
1935         const struct rte_flow_item_ipv6 *mask = item->mask;
1936         const struct rte_flow_item_ipv6 *spec = item->spec;
1937         const struct rte_flow_item_ipv6 nic_mask = {
1938                 .hdr = {
1939                         .src_addr =
1940                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1941                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1942                         .dst_addr =
1943                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1944                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1945                         .vtc_flow = RTE_BE32(0xffffffff),
1946                         .proto = 0xff,
1947                 },
1948         };
1949         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1950         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1951                                       MLX5_FLOW_LAYER_OUTER_L3;
1952         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1953                                       MLX5_FLOW_LAYER_OUTER_L4;
1954         int ret;
1955         uint8_t next_proto = 0xFF;
1956         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1957                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1958                                   MLX5_FLOW_LAYER_INNER_VLAN);
1959
1960         if ((last_item & l2_vlan) && ether_type &&
1961             ether_type != RTE_ETHER_TYPE_IPV6)
1962                 return rte_flow_error_set(error, EINVAL,
1963                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1964                                           "IPv6 cannot follow L2/VLAN layer "
1965                                           "which ether type is not IPv6");
1966         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
1967                 if (mask && spec)
1968                         next_proto = mask->hdr.proto & spec->hdr.proto;
1969                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1970                         return rte_flow_error_set(error, EINVAL,
1971                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1972                                                   item,
1973                                                   "multiple tunnel "
1974                                                   "not supported");
1975         }
1976         if (item_flags & MLX5_FLOW_LAYER_IPIP)
1977                 return rte_flow_error_set(error, EINVAL,
1978                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1979                                           "wrong tunnel type - IPv4 specified "
1980                                           "but IPv6 item provided");
1981         if (item_flags & l3m)
1982                 return rte_flow_error_set(error, ENOTSUP,
1983                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1984                                           "multiple L3 layers not supported");
1985         else if (item_flags & l4m)
1986                 return rte_flow_error_set(error, EINVAL,
1987                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1988                                           "L3 cannot follow an L4 layer.");
1989         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1990                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1991                 return rte_flow_error_set(error, EINVAL,
1992                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1993                                           "L3 cannot follow an NVGRE layer.");
1994         if (!mask)
1995                 mask = &rte_flow_item_ipv6_mask;
1996         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1997                                         acc_mask ? (const uint8_t *)acc_mask
1998                                                  : (const uint8_t *)&nic_mask,
1999                                         sizeof(struct rte_flow_item_ipv6),
2000                                         error);
2001         if (ret < 0)
2002                 return ret;
2003         return 0;
2004 }
2005
2006 /**
2007  * Validate UDP item.
2008  *
2009  * @param[in] item
2010  *   Item specification.
2011  * @param[in] item_flags
2012  *   Bit-fields that holds the items detected until now.
2013  * @param[in] target_protocol
2014  *   The next protocol in the previous item.
2015  * @param[in] flow_mask
2016  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2017  * @param[out] error
2018  *   Pointer to error structure.
2019  *
2020  * @return
2021  *   0 on success, a negative errno value otherwise and rte_errno is set.
2022  */
2023 int
2024 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2025                             uint64_t item_flags,
2026                             uint8_t target_protocol,
2027                             struct rte_flow_error *error)
2028 {
2029         const struct rte_flow_item_udp *mask = item->mask;
2030         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2031         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2032                                       MLX5_FLOW_LAYER_OUTER_L3;
2033         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2034                                       MLX5_FLOW_LAYER_OUTER_L4;
2035         int ret;
2036
2037         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2038                 return rte_flow_error_set(error, EINVAL,
2039                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2040                                           "protocol filtering not compatible"
2041                                           " with UDP layer");
2042         if (!(item_flags & l3m))
2043                 return rte_flow_error_set(error, EINVAL,
2044                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2045                                           "L3 is mandatory to filter on L4");
2046         if (item_flags & l4m)
2047                 return rte_flow_error_set(error, EINVAL,
2048                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2049                                           "multiple L4 layers not supported");
2050         if (!mask)
2051                 mask = &rte_flow_item_udp_mask;
2052         ret = mlx5_flow_item_acceptable
2053                 (item, (const uint8_t *)mask,
2054                  (const uint8_t *)&rte_flow_item_udp_mask,
2055                  sizeof(struct rte_flow_item_udp), error);
2056         if (ret < 0)
2057                 return ret;
2058         return 0;
2059 }
2060
2061 /**
2062  * Validate TCP item.
2063  *
2064  * @param[in] item
2065  *   Item specification.
2066  * @param[in] item_flags
2067  *   Bit-fields that holds the items detected until now.
2068  * @param[in] target_protocol
2069  *   The next protocol in the previous item.
2070  * @param[out] error
2071  *   Pointer to error structure.
2072  *
2073  * @return
2074  *   0 on success, a negative errno value otherwise and rte_errno is set.
2075  */
2076 int
2077 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2078                             uint64_t item_flags,
2079                             uint8_t target_protocol,
2080                             const struct rte_flow_item_tcp *flow_mask,
2081                             struct rte_flow_error *error)
2082 {
2083         const struct rte_flow_item_tcp *mask = item->mask;
2084         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2085         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2086                                       MLX5_FLOW_LAYER_OUTER_L3;
2087         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2088                                       MLX5_FLOW_LAYER_OUTER_L4;
2089         int ret;
2090
2091         MLX5_ASSERT(flow_mask);
2092         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2093                 return rte_flow_error_set(error, EINVAL,
2094                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2095                                           "protocol filtering not compatible"
2096                                           " with TCP layer");
2097         if (!(item_flags & l3m))
2098                 return rte_flow_error_set(error, EINVAL,
2099                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2100                                           "L3 is mandatory to filter on L4");
2101         if (item_flags & l4m)
2102                 return rte_flow_error_set(error, EINVAL,
2103                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2104                                           "multiple L4 layers not supported");
2105         if (!mask)
2106                 mask = &rte_flow_item_tcp_mask;
2107         ret = mlx5_flow_item_acceptable
2108                 (item, (const uint8_t *)mask,
2109                  (const uint8_t *)flow_mask,
2110                  sizeof(struct rte_flow_item_tcp), error);
2111         if (ret < 0)
2112                 return ret;
2113         return 0;
2114 }
2115
2116 /**
2117  * Validate VXLAN item.
2118  *
2119  * @param[in] item
2120  *   Item specification.
2121  * @param[in] item_flags
2122  *   Bit-fields that holds the items detected until now.
2123  * @param[in] target_protocol
2124  *   The next protocol in the previous item.
2125  * @param[out] error
2126  *   Pointer to error structure.
2127  *
2128  * @return
2129  *   0 on success, a negative errno value otherwise and rte_errno is set.
2130  */
2131 int
2132 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2133                               uint64_t item_flags,
2134                               struct rte_flow_error *error)
2135 {
2136         const struct rte_flow_item_vxlan *spec = item->spec;
2137         const struct rte_flow_item_vxlan *mask = item->mask;
2138         int ret;
2139         union vni {
2140                 uint32_t vlan_id;
2141                 uint8_t vni[4];
2142         } id = { .vlan_id = 0, };
2143
2144
2145         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2146                 return rte_flow_error_set(error, ENOTSUP,
2147                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2148                                           "multiple tunnel layers not"
2149                                           " supported");
2150         /*
2151          * Verify only UDPv4 is present as defined in
2152          * https://tools.ietf.org/html/rfc7348
2153          */
2154         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2155                 return rte_flow_error_set(error, EINVAL,
2156                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2157                                           "no outer UDP layer found");
2158         if (!mask)
2159                 mask = &rte_flow_item_vxlan_mask;
2160         ret = mlx5_flow_item_acceptable
2161                 (item, (const uint8_t *)mask,
2162                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2163                  sizeof(struct rte_flow_item_vxlan),
2164                  error);
2165         if (ret < 0)
2166                 return ret;
2167         if (spec) {
2168                 memcpy(&id.vni[1], spec->vni, 3);
2169                 memcpy(&id.vni[1], mask->vni, 3);
2170         }
2171         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2172                 return rte_flow_error_set(error, ENOTSUP,
2173                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2174                                           "VXLAN tunnel must be fully defined");
2175         return 0;
2176 }
2177
2178 /**
2179  * Validate VXLAN_GPE item.
2180  *
2181  * @param[in] item
2182  *   Item specification.
2183  * @param[in] item_flags
2184  *   Bit-fields that holds the items detected until now.
2185  * @param[in] priv
2186  *   Pointer to the private data structure.
2187  * @param[in] target_protocol
2188  *   The next protocol in the previous item.
2189  * @param[out] error
2190  *   Pointer to error structure.
2191  *
2192  * @return
2193  *   0 on success, a negative errno value otherwise and rte_errno is set.
2194  */
2195 int
2196 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2197                                   uint64_t item_flags,
2198                                   struct rte_eth_dev *dev,
2199                                   struct rte_flow_error *error)
2200 {
2201         struct mlx5_priv *priv = dev->data->dev_private;
2202         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2203         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2204         int ret;
2205         union vni {
2206                 uint32_t vlan_id;
2207                 uint8_t vni[4];
2208         } id = { .vlan_id = 0, };
2209
2210         if (!priv->config.l3_vxlan_en)
2211                 return rte_flow_error_set(error, ENOTSUP,
2212                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2213                                           "L3 VXLAN is not enabled by device"
2214                                           " parameter and/or not configured in"
2215                                           " firmware");
2216         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2217                 return rte_flow_error_set(error, ENOTSUP,
2218                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2219                                           "multiple tunnel layers not"
2220                                           " supported");
2221         /*
2222          * Verify only UDPv4 is present as defined in
2223          * https://tools.ietf.org/html/rfc7348
2224          */
2225         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2228                                           "no outer UDP layer found");
2229         if (!mask)
2230                 mask = &rte_flow_item_vxlan_gpe_mask;
2231         ret = mlx5_flow_item_acceptable
2232                 (item, (const uint8_t *)mask,
2233                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2234                  sizeof(struct rte_flow_item_vxlan_gpe),
2235                  error);
2236         if (ret < 0)
2237                 return ret;
2238         if (spec) {
2239                 if (spec->protocol)
2240                         return rte_flow_error_set(error, ENOTSUP,
2241                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2242                                                   item,
2243                                                   "VxLAN-GPE protocol"
2244                                                   " not supported");
2245                 memcpy(&id.vni[1], spec->vni, 3);
2246                 memcpy(&id.vni[1], mask->vni, 3);
2247         }
2248         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2249                 return rte_flow_error_set(error, ENOTSUP,
2250                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2251                                           "VXLAN-GPE tunnel must be fully"
2252                                           " defined");
2253         return 0;
2254 }
2255 /**
2256  * Validate GRE Key item.
2257  *
2258  * @param[in] item
2259  *   Item specification.
2260  * @param[in] item_flags
2261  *   Bit flags to mark detected items.
2262  * @param[in] gre_item
2263  *   Pointer to gre_item
2264  * @param[out] error
2265  *   Pointer to error structure.
2266  *
2267  * @return
2268  *   0 on success, a negative errno value otherwise and rte_errno is set.
2269  */
2270 int
2271 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2272                                 uint64_t item_flags,
2273                                 const struct rte_flow_item *gre_item,
2274                                 struct rte_flow_error *error)
2275 {
2276         const rte_be32_t *mask = item->mask;
2277         int ret = 0;
2278         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2279         const struct rte_flow_item_gre *gre_spec;
2280         const struct rte_flow_item_gre *gre_mask;
2281
2282         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2283                 return rte_flow_error_set(error, ENOTSUP,
2284                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2285                                           "Multiple GRE key not support");
2286         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2287                 return rte_flow_error_set(error, ENOTSUP,
2288                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2289                                           "No preceding GRE header");
2290         if (item_flags & MLX5_FLOW_LAYER_INNER)
2291                 return rte_flow_error_set(error, ENOTSUP,
2292                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2293                                           "GRE key following a wrong item");
2294         gre_mask = gre_item->mask;
2295         if (!gre_mask)
2296                 gre_mask = &rte_flow_item_gre_mask;
2297         gre_spec = gre_item->spec;
2298         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2299                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2300                 return rte_flow_error_set(error, EINVAL,
2301                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2302                                           "Key bit must be on");
2303
2304         if (!mask)
2305                 mask = &gre_key_default_mask;
2306         ret = mlx5_flow_item_acceptable
2307                 (item, (const uint8_t *)mask,
2308                  (const uint8_t *)&gre_key_default_mask,
2309                  sizeof(rte_be32_t), error);
2310         return ret;
2311 }
2312
2313 /**
2314  * Validate GRE item.
2315  *
2316  * @param[in] item
2317  *   Item specification.
2318  * @param[in] item_flags
2319  *   Bit flags to mark detected items.
2320  * @param[in] target_protocol
2321  *   The next protocol in the previous item.
2322  * @param[out] error
2323  *   Pointer to error structure.
2324  *
2325  * @return
2326  *   0 on success, a negative errno value otherwise and rte_errno is set.
2327  */
2328 int
2329 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2330                             uint64_t item_flags,
2331                             uint8_t target_protocol,
2332                             struct rte_flow_error *error)
2333 {
2334         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2335         const struct rte_flow_item_gre *mask = item->mask;
2336         int ret;
2337         const struct rte_flow_item_gre nic_mask = {
2338                 .c_rsvd0_ver = RTE_BE16(0xB000),
2339                 .protocol = RTE_BE16(UINT16_MAX),
2340         };
2341
2342         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2343                 return rte_flow_error_set(error, EINVAL,
2344                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2345                                           "protocol filtering not compatible"
2346                                           " with this GRE layer");
2347         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2348                 return rte_flow_error_set(error, ENOTSUP,
2349                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2350                                           "multiple tunnel layers not"
2351                                           " supported");
2352         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2353                 return rte_flow_error_set(error, ENOTSUP,
2354                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2355                                           "L3 Layer is missing");
2356         if (!mask)
2357                 mask = &rte_flow_item_gre_mask;
2358         ret = mlx5_flow_item_acceptable
2359                 (item, (const uint8_t *)mask,
2360                  (const uint8_t *)&nic_mask,
2361                  sizeof(struct rte_flow_item_gre), error);
2362         if (ret < 0)
2363                 return ret;
2364 #ifndef HAVE_MLX5DV_DR
2365 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2366         if (spec && (spec->protocol & mask->protocol))
2367                 return rte_flow_error_set(error, ENOTSUP,
2368                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2369                                           "without MPLS support the"
2370                                           " specification cannot be used for"
2371                                           " filtering");
2372 #endif
2373 #endif
2374         return 0;
2375 }
2376
2377 /**
2378  * Validate Geneve item.
2379  *
2380  * @param[in] item
2381  *   Item specification.
2382  * @param[in] itemFlags
2383  *   Bit-fields that holds the items detected until now.
2384  * @param[in] enPriv
2385  *   Pointer to the private data structure.
2386  * @param[out] error
2387  *   Pointer to error structure.
2388  *
2389  * @return
2390  *   0 on success, a negative errno value otherwise and rte_errno is set.
2391  */
2392
2393 int
2394 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2395                                uint64_t item_flags,
2396                                struct rte_eth_dev *dev,
2397                                struct rte_flow_error *error)
2398 {
2399         struct mlx5_priv *priv = dev->data->dev_private;
2400         const struct rte_flow_item_geneve *spec = item->spec;
2401         const struct rte_flow_item_geneve *mask = item->mask;
2402         int ret;
2403         uint16_t gbhdr;
2404         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2405                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2406         const struct rte_flow_item_geneve nic_mask = {
2407                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2408                 .vni = "\xff\xff\xff",
2409                 .protocol = RTE_BE16(UINT16_MAX),
2410         };
2411
2412         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2413                 return rte_flow_error_set(error, ENOTSUP,
2414                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2415                                           "L3 Geneve is not enabled by device"
2416                                           " parameter and/or not configured in"
2417                                           " firmware");
2418         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2419                 return rte_flow_error_set(error, ENOTSUP,
2420                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2421                                           "multiple tunnel layers not"
2422                                           " supported");
2423         /*
2424          * Verify only UDPv4 is present as defined in
2425          * https://tools.ietf.org/html/rfc7348
2426          */
2427         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2428                 return rte_flow_error_set(error, EINVAL,
2429                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2430                                           "no outer UDP layer found");
2431         if (!mask)
2432                 mask = &rte_flow_item_geneve_mask;
2433         ret = mlx5_flow_item_acceptable
2434                                   (item, (const uint8_t *)mask,
2435                                    (const uint8_t *)&nic_mask,
2436                                    sizeof(struct rte_flow_item_geneve), error);
2437         if (ret)
2438                 return ret;
2439         if (spec) {
2440                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2441                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2442                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2443                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2444                         return rte_flow_error_set(error, ENOTSUP,
2445                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2446                                                   item,
2447                                                   "Geneve protocol unsupported"
2448                                                   " fields are being used");
2449                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2450                         return rte_flow_error_set
2451                                         (error, ENOTSUP,
2452                                          RTE_FLOW_ERROR_TYPE_ITEM,
2453                                          item,
2454                                          "Unsupported Geneve options length");
2455         }
2456         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2457                 return rte_flow_error_set
2458                                     (error, ENOTSUP,
2459                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2460                                      "Geneve tunnel must be fully defined");
2461         return 0;
2462 }
2463
2464 /**
2465  * Validate MPLS item.
2466  *
2467  * @param[in] dev
2468  *   Pointer to the rte_eth_dev structure.
2469  * @param[in] item
2470  *   Item specification.
2471  * @param[in] item_flags
2472  *   Bit-fields that holds the items detected until now.
2473  * @param[in] prev_layer
2474  *   The protocol layer indicated in previous item.
2475  * @param[out] error
2476  *   Pointer to error structure.
2477  *
2478  * @return
2479  *   0 on success, a negative errno value otherwise and rte_errno is set.
2480  */
2481 int
2482 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2483                              const struct rte_flow_item *item __rte_unused,
2484                              uint64_t item_flags __rte_unused,
2485                              uint64_t prev_layer __rte_unused,
2486                              struct rte_flow_error *error)
2487 {
2488 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2489         const struct rte_flow_item_mpls *mask = item->mask;
2490         struct mlx5_priv *priv = dev->data->dev_private;
2491         int ret;
2492
2493         if (!priv->config.mpls_en)
2494                 return rte_flow_error_set(error, ENOTSUP,
2495                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2496                                           "MPLS not supported or"
2497                                           " disabled in firmware"
2498                                           " configuration.");
2499         /* MPLS over IP, UDP, GRE is allowed */
2500         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2501                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2502                             MLX5_FLOW_LAYER_GRE)))
2503                 return rte_flow_error_set(error, EINVAL,
2504                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2505                                           "protocol filtering not compatible"
2506                                           " with MPLS layer");
2507         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2508         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2509             !(item_flags & MLX5_FLOW_LAYER_GRE))
2510                 return rte_flow_error_set(error, ENOTSUP,
2511                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2512                                           "multiple tunnel layers not"
2513                                           " supported");
2514         if (!mask)
2515                 mask = &rte_flow_item_mpls_mask;
2516         ret = mlx5_flow_item_acceptable
2517                 (item, (const uint8_t *)mask,
2518                  (const uint8_t *)&rte_flow_item_mpls_mask,
2519                  sizeof(struct rte_flow_item_mpls), error);
2520         if (ret < 0)
2521                 return ret;
2522         return 0;
2523 #else
2524         return rte_flow_error_set(error, ENOTSUP,
2525                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2526                                   "MPLS is not supported by Verbs, please"
2527                                   " update.");
2528 #endif
2529 }
2530
2531 /**
2532  * Validate NVGRE item.
2533  *
2534  * @param[in] item
2535  *   Item specification.
2536  * @param[in] item_flags
2537  *   Bit flags to mark detected items.
2538  * @param[in] target_protocol
2539  *   The next protocol in the previous item.
2540  * @param[out] error
2541  *   Pointer to error structure.
2542  *
2543  * @return
2544  *   0 on success, a negative errno value otherwise and rte_errno is set.
2545  */
2546 int
2547 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2548                               uint64_t item_flags,
2549                               uint8_t target_protocol,
2550                               struct rte_flow_error *error)
2551 {
2552         const struct rte_flow_item_nvgre *mask = item->mask;
2553         int ret;
2554
2555         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2556                 return rte_flow_error_set(error, EINVAL,
2557                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2558                                           "protocol filtering not compatible"
2559                                           " with this GRE layer");
2560         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2561                 return rte_flow_error_set(error, ENOTSUP,
2562                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2563                                           "multiple tunnel layers not"
2564                                           " supported");
2565         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2566                 return rte_flow_error_set(error, ENOTSUP,
2567                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2568                                           "L3 Layer is missing");
2569         if (!mask)
2570                 mask = &rte_flow_item_nvgre_mask;
2571         ret = mlx5_flow_item_acceptable
2572                 (item, (const uint8_t *)mask,
2573                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2574                  sizeof(struct rte_flow_item_nvgre), error);
2575         if (ret < 0)
2576                 return ret;
2577         return 0;
2578 }
2579
2580 /**
2581  * Validate eCPRI item.
2582  *
2583  * @param[in] item
2584  *   Item specification.
2585  * @param[in] item_flags
2586  *   Bit-fields that holds the items detected until now.
2587  * @param[in] last_item
2588  *   Previous validated item in the pattern items.
2589  * @param[in] ether_type
2590  *   Type in the ethernet layer header (including dot1q).
2591  * @param[in] acc_mask
2592  *   Acceptable mask, if NULL default internal default mask
2593  *   will be used to check whether item fields are supported.
2594  * @param[out] error
2595  *   Pointer to error structure.
2596  *
2597  * @return
2598  *   0 on success, a negative errno value otherwise and rte_errno is set.
2599  */
2600 int
2601 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2602                               uint64_t item_flags,
2603                               uint64_t last_item,
2604                               uint16_t ether_type,
2605                               const struct rte_flow_item_ecpri *acc_mask,
2606                               struct rte_flow_error *error)
2607 {
2608         const struct rte_flow_item_ecpri *mask = item->mask;
2609         const struct rte_flow_item_ecpri nic_mask = {
2610                 .hdr = {
2611                         .common = {
2612                                 .u32 =
2613                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2614                                         .type = 0xFF,
2615                                         }).u32),
2616                         },
2617                         .dummy[0] = 0xFFFFFFFF,
2618                 },
2619         };
2620         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2621                                         MLX5_FLOW_LAYER_OUTER_VLAN);
2622         struct rte_flow_item_ecpri mask_lo;
2623
2624         if ((last_item & outer_l2_vlan) && ether_type &&
2625             ether_type != RTE_ETHER_TYPE_ECPRI)
2626                 return rte_flow_error_set(error, EINVAL,
2627                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2628                                           "eCPRI cannot follow L2/VLAN layer "
2629                                           "which ether type is not 0xAEFE.");
2630         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2631                 return rte_flow_error_set(error, EINVAL,
2632                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2633                                           "eCPRI with tunnel is not supported "
2634                                           "right now.");
2635         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2636                 return rte_flow_error_set(error, ENOTSUP,
2637                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2638                                           "multiple L3 layers not supported");
2639         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2640                 return rte_flow_error_set(error, EINVAL,
2641                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2642                                           "eCPRI cannot follow a TCP layer.");
2643         /* In specification, eCPRI could be over UDP layer. */
2644         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2645                 return rte_flow_error_set(error, EINVAL,
2646                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2647                                           "eCPRI over UDP layer is not yet "
2648                                           "supported right now.");
2649         /* Mask for type field in common header could be zero. */
2650         if (!mask)
2651                 mask = &rte_flow_item_ecpri_mask;
2652         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2653         /* Input mask is in big-endian format. */
2654         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2655                 return rte_flow_error_set(error, EINVAL,
2656                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2657                                           "partial mask is not supported "
2658                                           "for protocol");
2659         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2660                 return rte_flow_error_set(error, EINVAL,
2661                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2662                                           "message header mask must be after "
2663                                           "a type mask");
2664         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2665                                          acc_mask ? (const uint8_t *)acc_mask
2666                                                   : (const uint8_t *)&nic_mask,
2667                                          sizeof(struct rte_flow_item_ecpri),
2668                                          error);
2669 }
2670
2671 /* Allocate unique ID for the split Q/RSS subflows. */
2672 static uint32_t
2673 flow_qrss_get_id(struct rte_eth_dev *dev)
2674 {
2675         struct mlx5_priv *priv = dev->data->dev_private;
2676         uint32_t qrss_id, ret;
2677
2678         ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2679         if (ret)
2680                 return 0;
2681         MLX5_ASSERT(qrss_id);
2682         return qrss_id;
2683 }
2684
2685 /* Free unique ID for the split Q/RSS subflows. */
2686 static void
2687 flow_qrss_free_id(struct rte_eth_dev *dev,  uint32_t qrss_id)
2688 {
2689         struct mlx5_priv *priv = dev->data->dev_private;
2690
2691         if (qrss_id)
2692                 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2693 }
2694
2695 /**
2696  * Release resource related QUEUE/RSS action split.
2697  *
2698  * @param dev
2699  *   Pointer to Ethernet device.
2700  * @param flow
2701  *   Flow to release id's from.
2702  */
2703 static void
2704 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2705                              struct rte_flow *flow)
2706 {
2707         struct mlx5_priv *priv = dev->data->dev_private;
2708         uint32_t handle_idx;
2709         struct mlx5_flow_handle *dev_handle;
2710
2711         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2712                        handle_idx, dev_handle, next)
2713                 if (dev_handle->split_flow_id)
2714                         flow_qrss_free_id(dev, dev_handle->split_flow_id);
2715 }
2716
2717 static int
2718 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2719                    const struct rte_flow_attr *attr __rte_unused,
2720                    const struct rte_flow_item items[] __rte_unused,
2721                    const struct rte_flow_action actions[] __rte_unused,
2722                    bool external __rte_unused,
2723                    int hairpin __rte_unused,
2724                    struct rte_flow_error *error)
2725 {
2726         return rte_flow_error_set(error, ENOTSUP,
2727                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2728 }
2729
2730 static struct mlx5_flow *
2731 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2732                   const struct rte_flow_attr *attr __rte_unused,
2733                   const struct rte_flow_item items[] __rte_unused,
2734                   const struct rte_flow_action actions[] __rte_unused,
2735                   struct rte_flow_error *error)
2736 {
2737         rte_flow_error_set(error, ENOTSUP,
2738                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2739         return NULL;
2740 }
2741
2742 static int
2743 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2744                     struct mlx5_flow *dev_flow __rte_unused,
2745                     const struct rte_flow_attr *attr __rte_unused,
2746                     const struct rte_flow_item items[] __rte_unused,
2747                     const struct rte_flow_action actions[] __rte_unused,
2748                     struct rte_flow_error *error)
2749 {
2750         return rte_flow_error_set(error, ENOTSUP,
2751                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2752 }
2753
2754 static int
2755 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2756                 struct rte_flow *flow __rte_unused,
2757                 struct rte_flow_error *error)
2758 {
2759         return rte_flow_error_set(error, ENOTSUP,
2760                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2761 }
2762
2763 static void
2764 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2765                  struct rte_flow *flow __rte_unused)
2766 {
2767 }
2768
2769 static void
2770 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2771                   struct rte_flow *flow __rte_unused)
2772 {
2773 }
2774
2775 static int
2776 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2777                 struct rte_flow *flow __rte_unused,
2778                 const struct rte_flow_action *actions __rte_unused,
2779                 void *data __rte_unused,
2780                 struct rte_flow_error *error)
2781 {
2782         return rte_flow_error_set(error, ENOTSUP,
2783                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2784 }
2785
2786 /* Void driver to protect from null pointer reference. */
2787 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2788         .validate = flow_null_validate,
2789         .prepare = flow_null_prepare,
2790         .translate = flow_null_translate,
2791         .apply = flow_null_apply,
2792         .remove = flow_null_remove,
2793         .destroy = flow_null_destroy,
2794         .query = flow_null_query,
2795 };
2796
2797 /**
2798  * Select flow driver type according to flow attributes and device
2799  * configuration.
2800  *
2801  * @param[in] dev
2802  *   Pointer to the dev structure.
2803  * @param[in] attr
2804  *   Pointer to the flow attributes.
2805  *
2806  * @return
2807  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2808  */
2809 static enum mlx5_flow_drv_type
2810 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2811 {
2812         struct mlx5_priv *priv = dev->data->dev_private;
2813         /* The OS can determine first a specific flow type (DV, VERBS) */
2814         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2815
2816         if (type != MLX5_FLOW_TYPE_MAX)
2817                 return type;
2818         /* If no OS specific type - continue with DV/VERBS selection */
2819         if (attr->transfer && priv->config.dv_esw_en)
2820                 type = MLX5_FLOW_TYPE_DV;
2821         if (!attr->transfer)
2822                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2823                                                  MLX5_FLOW_TYPE_VERBS;
2824         return type;
2825 }
2826
2827 #define flow_get_drv_ops(type) flow_drv_ops[type]
2828
2829 /**
2830  * Flow driver validation API. This abstracts calling driver specific functions.
2831  * The type of flow driver is determined according to flow attributes.
2832  *
2833  * @param[in] dev
2834  *   Pointer to the dev structure.
2835  * @param[in] attr
2836  *   Pointer to the flow attributes.
2837  * @param[in] items
2838  *   Pointer to the list of items.
2839  * @param[in] actions
2840  *   Pointer to the list of actions.
2841  * @param[in] external
2842  *   This flow rule is created by request external to PMD.
2843  * @param[in] hairpin
2844  *   Number of hairpin TX actions, 0 means classic flow.
2845  * @param[out] error
2846  *   Pointer to the error structure.
2847  *
2848  * @return
2849  *   0 on success, a negative errno value otherwise and rte_errno is set.
2850  */
2851 static inline int
2852 flow_drv_validate(struct rte_eth_dev *dev,
2853                   const struct rte_flow_attr *attr,
2854                   const struct rte_flow_item items[],
2855                   const struct rte_flow_action actions[],
2856                   bool external, int hairpin, struct rte_flow_error *error)
2857 {
2858         const struct mlx5_flow_driver_ops *fops;
2859         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2860
2861         fops = flow_get_drv_ops(type);
2862         return fops->validate(dev, attr, items, actions, external,
2863                               hairpin, error);
2864 }
2865
2866 /**
2867  * Flow driver preparation API. This abstracts calling driver specific
2868  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2869  * calculates the size of memory required for device flow, allocates the memory,
2870  * initializes the device flow and returns the pointer.
2871  *
2872  * @note
2873  *   This function initializes device flow structure such as dv or verbs in
2874  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
2875  *   rest. For example, adding returning device flow to flow->dev_flow list and
2876  *   setting backward reference to the flow should be done out of this function.
2877  *   layers field is not filled either.
2878  *
2879  * @param[in] dev
2880  *   Pointer to the dev structure.
2881  * @param[in] attr
2882  *   Pointer to the flow attributes.
2883  * @param[in] items
2884  *   Pointer to the list of items.
2885  * @param[in] actions
2886  *   Pointer to the list of actions.
2887  * @param[in] flow_idx
2888  *   This memory pool index to the flow.
2889  * @param[out] error
2890  *   Pointer to the error structure.
2891  *
2892  * @return
2893  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
2894  */
2895 static inline struct mlx5_flow *
2896 flow_drv_prepare(struct rte_eth_dev *dev,
2897                  const struct rte_flow *flow,
2898                  const struct rte_flow_attr *attr,
2899                  const struct rte_flow_item items[],
2900                  const struct rte_flow_action actions[],
2901                  uint32_t flow_idx,
2902                  struct rte_flow_error *error)
2903 {
2904         const struct mlx5_flow_driver_ops *fops;
2905         enum mlx5_flow_drv_type type = flow->drv_type;
2906         struct mlx5_flow *mlx5_flow = NULL;
2907
2908         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2909         fops = flow_get_drv_ops(type);
2910         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
2911         if (mlx5_flow)
2912                 mlx5_flow->flow_idx = flow_idx;
2913         return mlx5_flow;
2914 }
2915
2916 /**
2917  * Flow driver translation API. This abstracts calling driver specific
2918  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2919  * translates a generic flow into a driver flow. flow_drv_prepare() must
2920  * precede.
2921  *
2922  * @note
2923  *   dev_flow->layers could be filled as a result of parsing during translation
2924  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
2925  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
2926  *   flow->actions could be overwritten even though all the expanded dev_flows
2927  *   have the same actions.
2928  *
2929  * @param[in] dev
2930  *   Pointer to the rte dev structure.
2931  * @param[in, out] dev_flow
2932  *   Pointer to the mlx5 flow.
2933  * @param[in] attr
2934  *   Pointer to the flow attributes.
2935  * @param[in] items
2936  *   Pointer to the list of items.
2937  * @param[in] actions
2938  *   Pointer to the list of actions.
2939  * @param[out] error
2940  *   Pointer to the error structure.
2941  *
2942  * @return
2943  *   0 on success, a negative errno value otherwise and rte_errno is set.
2944  */
2945 static inline int
2946 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2947                    const struct rte_flow_attr *attr,
2948                    const struct rte_flow_item items[],
2949                    const struct rte_flow_action actions[],
2950                    struct rte_flow_error *error)
2951 {
2952         const struct mlx5_flow_driver_ops *fops;
2953         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
2954
2955         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2956         fops = flow_get_drv_ops(type);
2957         return fops->translate(dev, dev_flow, attr, items, actions, error);
2958 }
2959
2960 /**
2961  * Flow driver apply API. This abstracts calling driver specific functions.
2962  * Parent flow (rte_flow) should have driver type (drv_type). It applies
2963  * translated driver flows on to device. flow_drv_translate() must precede.
2964  *
2965  * @param[in] dev
2966  *   Pointer to Ethernet device structure.
2967  * @param[in, out] flow
2968  *   Pointer to flow structure.
2969  * @param[out] error
2970  *   Pointer to error structure.
2971  *
2972  * @return
2973  *   0 on success, a negative errno value otherwise and rte_errno is set.
2974  */
2975 static inline int
2976 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
2977                struct rte_flow_error *error)
2978 {
2979         const struct mlx5_flow_driver_ops *fops;
2980         enum mlx5_flow_drv_type type = flow->drv_type;
2981
2982         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2983         fops = flow_get_drv_ops(type);
2984         return fops->apply(dev, flow, error);
2985 }
2986
2987 /**
2988  * Flow driver remove API. This abstracts calling driver specific functions.
2989  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2990  * on device. All the resources of the flow should be freed by calling
2991  * flow_drv_destroy().
2992  *
2993  * @param[in] dev
2994  *   Pointer to Ethernet device.
2995  * @param[in, out] flow
2996  *   Pointer to flow structure.
2997  */
2998 static inline void
2999 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
3000 {
3001         const struct mlx5_flow_driver_ops *fops;
3002         enum mlx5_flow_drv_type type = flow->drv_type;
3003
3004         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3005         fops = flow_get_drv_ops(type);
3006         fops->remove(dev, flow);
3007 }
3008
3009 /**
3010  * Flow driver destroy API. This abstracts calling driver specific functions.
3011  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3012  * on device and releases resources of the flow.
3013  *
3014  * @param[in] dev
3015  *   Pointer to Ethernet device.
3016  * @param[in, out] flow
3017  *   Pointer to flow structure.
3018  */
3019 static inline void
3020 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3021 {
3022         const struct mlx5_flow_driver_ops *fops;
3023         enum mlx5_flow_drv_type type = flow->drv_type;
3024
3025         flow_mreg_split_qrss_release(dev, flow);
3026         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3027         fops = flow_get_drv_ops(type);
3028         fops->destroy(dev, flow);
3029 }
3030
3031 /**
3032  * Get RSS action from the action list.
3033  *
3034  * @param[in] actions
3035  *   Pointer to the list of actions.
3036  *
3037  * @return
3038  *   Pointer to the RSS action if exist, else return NULL.
3039  */
3040 static const struct rte_flow_action_rss*
3041 flow_get_rss_action(const struct rte_flow_action actions[])
3042 {
3043         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3044                 switch (actions->type) {
3045                 case RTE_FLOW_ACTION_TYPE_RSS:
3046                         return (const struct rte_flow_action_rss *)
3047                                actions->conf;
3048                 default:
3049                         break;
3050                 }
3051         }
3052         return NULL;
3053 }
3054
3055 static unsigned int
3056 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3057 {
3058         const struct rte_flow_item *item;
3059         unsigned int has_vlan = 0;
3060
3061         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3062                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3063                         has_vlan = 1;
3064                         break;
3065                 }
3066         }
3067         if (has_vlan)
3068                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3069                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3070         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3071                                MLX5_EXPANSION_ROOT_OUTER;
3072 }
3073
3074 /**
3075  *  Get layer flags from the prefix flow.
3076  *
3077  *  Some flows may be split to several subflows, the prefix subflow gets the
3078  *  match items and the suffix sub flow gets the actions.
3079  *  Some actions need the user defined match item flags to get the detail for
3080  *  the action.
3081  *  This function helps the suffix flow to get the item layer flags from prefix
3082  *  subflow.
3083  *
3084  * @param[in] dev_flow
3085  *   Pointer the created preifx subflow.
3086  *
3087  * @return
3088  *   The layers get from prefix subflow.
3089  */
3090 static inline uint64_t
3091 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3092 {
3093         uint64_t layers = 0;
3094
3095         /*
3096          * Layers bits could be localization, but usually the compiler will
3097          * help to do the optimization work for source code.
3098          * If no decap actions, use the layers directly.
3099          */
3100         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3101                 return dev_flow->handle->layers;
3102         /* Convert L3 layers with decap action. */
3103         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3104                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3105         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3106                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3107         /* Convert L4 layers with decap action.  */
3108         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3109                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3110         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3111                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3112         return layers;
3113 }
3114
3115 /**
3116  * Get metadata split action information.
3117  *
3118  * @param[in] actions
3119  *   Pointer to the list of actions.
3120  * @param[out] qrss
3121  *   Pointer to the return pointer.
3122  * @param[out] qrss_type
3123  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3124  *   if no QUEUE/RSS is found.
3125  * @param[out] encap_idx
3126  *   Pointer to the index of the encap action if exists, otherwise the last
3127  *   action index.
3128  *
3129  * @return
3130  *   Total number of actions.
3131  */
3132 static int
3133 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3134                                        const struct rte_flow_action **qrss,
3135                                        int *encap_idx)
3136 {
3137         const struct rte_flow_action_raw_encap *raw_encap;
3138         int actions_n = 0;
3139         int raw_decap_idx = -1;
3140
3141         *encap_idx = -1;
3142         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3143                 switch (actions->type) {
3144                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3145                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3146                         *encap_idx = actions_n;
3147                         break;
3148                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3149                         raw_decap_idx = actions_n;
3150                         break;
3151                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3152                         raw_encap = actions->conf;
3153                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3154                                 *encap_idx = raw_decap_idx != -1 ?
3155                                                       raw_decap_idx : actions_n;
3156                         break;
3157                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3158                 case RTE_FLOW_ACTION_TYPE_RSS:
3159                         *qrss = actions;
3160                         break;
3161                 default:
3162                         break;
3163                 }
3164                 actions_n++;
3165         }
3166         if (*encap_idx == -1)
3167                 *encap_idx = actions_n;
3168         /* Count RTE_FLOW_ACTION_TYPE_END. */
3169         return actions_n + 1;
3170 }
3171
3172 /**
3173  * Check meter action from the action list.
3174  *
3175  * @param[in] actions
3176  *   Pointer to the list of actions.
3177  * @param[out] mtr
3178  *   Pointer to the meter exist flag.
3179  *
3180  * @return
3181  *   Total number of actions.
3182  */
3183 static int
3184 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
3185 {
3186         int actions_n = 0;
3187
3188         MLX5_ASSERT(mtr);
3189         *mtr = 0;
3190         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3191                 switch (actions->type) {
3192                 case RTE_FLOW_ACTION_TYPE_METER:
3193                         *mtr = 1;
3194                         break;
3195                 default:
3196                         break;
3197                 }
3198                 actions_n++;
3199         }
3200         /* Count RTE_FLOW_ACTION_TYPE_END. */
3201         return actions_n + 1;
3202 }
3203
3204 /**
3205  * Check if the flow should be split due to hairpin.
3206  * The reason for the split is that in current HW we can't
3207  * support encap and push-vlan on Rx, so if a flow contains
3208  * these actions we move it to Tx.
3209  *
3210  * @param dev
3211  *   Pointer to Ethernet device.
3212  * @param[in] attr
3213  *   Flow rule attributes.
3214  * @param[in] actions
3215  *   Associated actions (list terminated by the END action).
3216  *
3217  * @return
3218  *   > 0 the number of actions and the flow should be split,
3219  *   0 when no split required.
3220  */
3221 static int
3222 flow_check_hairpin_split(struct rte_eth_dev *dev,
3223                          const struct rte_flow_attr *attr,
3224                          const struct rte_flow_action actions[])
3225 {
3226         int queue_action = 0;
3227         int action_n = 0;
3228         int split = 0;
3229         const struct rte_flow_action_queue *queue;
3230         const struct rte_flow_action_rss *rss;
3231         const struct rte_flow_action_raw_encap *raw_encap;
3232
3233         if (!attr->ingress)
3234                 return 0;
3235         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3236                 switch (actions->type) {
3237                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3238                         queue = actions->conf;
3239                         if (queue == NULL)
3240                                 return 0;
3241                         if (mlx5_rxq_get_type(dev, queue->index) !=
3242                             MLX5_RXQ_TYPE_HAIRPIN)
3243                                 return 0;
3244                         queue_action = 1;
3245                         action_n++;
3246                         break;
3247                 case RTE_FLOW_ACTION_TYPE_RSS:
3248                         rss = actions->conf;
3249                         if (rss == NULL || rss->queue_num == 0)
3250                                 return 0;
3251                         if (mlx5_rxq_get_type(dev, rss->queue[0]) !=
3252                             MLX5_RXQ_TYPE_HAIRPIN)
3253                                 return 0;
3254                         queue_action = 1;
3255                         action_n++;
3256                         break;
3257                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3258                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3259                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3260                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3261                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3262                         split++;
3263                         action_n++;
3264                         break;
3265                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3266                         raw_encap = actions->conf;
3267                         if (raw_encap->size >
3268                             (sizeof(struct rte_flow_item_eth) +
3269                              sizeof(struct rte_flow_item_ipv4)))
3270                                 split++;
3271                         action_n++;
3272                         break;
3273                 default:
3274                         action_n++;
3275                         break;
3276                 }
3277         }
3278         if (split && queue_action)
3279                 return action_n;
3280         return 0;
3281 }
3282
3283 /* Declare flow create/destroy prototype in advance. */
3284 static uint32_t
3285 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
3286                  const struct rte_flow_attr *attr,
3287                  const struct rte_flow_item items[],
3288                  const struct rte_flow_action actions[],
3289                  bool external, struct rte_flow_error *error);
3290
3291 static void
3292 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
3293                   uint32_t flow_idx);
3294
3295 /**
3296  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3297  *
3298  * As mark_id is unique, if there's already a registered flow for the mark_id,
3299  * return by increasing the reference counter of the resource. Otherwise, create
3300  * the resource (mcp_res) and flow.
3301  *
3302  * Flow looks like,
3303  *   - If ingress port is ANY and reg_c[1] is mark_id,
3304  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3305  *
3306  * For default flow (zero mark_id), flow is like,
3307  *   - If ingress port is ANY,
3308  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
3309  *
3310  * @param dev
3311  *   Pointer to Ethernet device.
3312  * @param mark_id
3313  *   ID of MARK action, zero means default flow for META.
3314  * @param[out] error
3315  *   Perform verbose error reporting if not NULL.
3316  *
3317  * @return
3318  *   Associated resource on success, NULL otherwise and rte_errno is set.
3319  */
3320 static struct mlx5_flow_mreg_copy_resource *
3321 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
3322                           struct rte_flow_error *error)
3323 {
3324         struct mlx5_priv *priv = dev->data->dev_private;
3325         struct rte_flow_attr attr = {
3326                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3327                 .ingress = 1,
3328         };
3329         struct mlx5_rte_flow_item_tag tag_spec = {
3330                 .data = mark_id,
3331         };
3332         struct rte_flow_item items[] = {
3333                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3334         };
3335         struct rte_flow_action_mark ftag = {
3336                 .id = mark_id,
3337         };
3338         struct mlx5_flow_action_copy_mreg cp_mreg = {
3339                 .dst = REG_B,
3340                 .src = REG_NON,
3341         };
3342         struct rte_flow_action_jump jump = {
3343                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3344         };
3345         struct rte_flow_action actions[] = {
3346                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3347         };
3348         struct mlx5_flow_mreg_copy_resource *mcp_res;
3349         uint32_t idx = 0;
3350         int ret;
3351
3352         /* Fill the register fileds in the flow. */
3353         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3354         if (ret < 0)
3355                 return NULL;
3356         tag_spec.id = ret;
3357         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3358         if (ret < 0)
3359                 return NULL;
3360         cp_mreg.src = ret;
3361         /* Check if already registered. */
3362         MLX5_ASSERT(priv->mreg_cp_tbl);
3363         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
3364         if (mcp_res) {
3365                 /* For non-default rule. */
3366                 if (mark_id != MLX5_DEFAULT_COPY_ID)
3367                         mcp_res->refcnt++;
3368                 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
3369                             mcp_res->refcnt == 1);
3370                 return mcp_res;
3371         }
3372         /* Provide the full width of FLAG specific value. */
3373         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3374                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3375         /* Build a new flow. */
3376         if (mark_id != MLX5_DEFAULT_COPY_ID) {
3377                 items[0] = (struct rte_flow_item){
3378                         .type = (enum rte_flow_item_type)
3379                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3380                         .spec = &tag_spec,
3381                 };
3382                 items[1] = (struct rte_flow_item){
3383                         .type = RTE_FLOW_ITEM_TYPE_END,
3384                 };
3385                 actions[0] = (struct rte_flow_action){
3386                         .type = (enum rte_flow_action_type)
3387                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3388                         .conf = &ftag,
3389                 };
3390                 actions[1] = (struct rte_flow_action){
3391                         .type = (enum rte_flow_action_type)
3392                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3393                         .conf = &cp_mreg,
3394                 };
3395                 actions[2] = (struct rte_flow_action){
3396                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3397                         .conf = &jump,
3398                 };
3399                 actions[3] = (struct rte_flow_action){
3400                         .type = RTE_FLOW_ACTION_TYPE_END,
3401                 };
3402         } else {
3403                 /* Default rule, wildcard match. */
3404                 attr.priority = MLX5_FLOW_PRIO_RSVD;
3405                 items[0] = (struct rte_flow_item){
3406                         .type = RTE_FLOW_ITEM_TYPE_END,
3407                 };
3408                 actions[0] = (struct rte_flow_action){
3409                         .type = (enum rte_flow_action_type)
3410                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3411                         .conf = &cp_mreg,
3412                 };
3413                 actions[1] = (struct rte_flow_action){
3414                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
3415                         .conf = &jump,
3416                 };
3417                 actions[2] = (struct rte_flow_action){
3418                         .type = RTE_FLOW_ACTION_TYPE_END,
3419                 };
3420         }
3421         /* Build a new entry. */
3422         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3423         if (!mcp_res) {
3424                 rte_errno = ENOMEM;
3425                 return NULL;
3426         }
3427         mcp_res->idx = idx;
3428         /*
3429          * The copy Flows are not included in any list. There
3430          * ones are referenced from other Flows and can not
3431          * be applied, removed, deleted in ardbitrary order
3432          * by list traversing.
3433          */
3434         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3435                                          actions, false, error);
3436         if (!mcp_res->rix_flow)
3437                 goto error;
3438         mcp_res->refcnt++;
3439         mcp_res->hlist_ent.key = mark_id;
3440         ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3441                                 &mcp_res->hlist_ent);
3442         MLX5_ASSERT(!ret);
3443         if (ret)
3444                 goto error;
3445         return mcp_res;
3446 error:
3447         if (mcp_res->rix_flow)
3448                 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3449         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3450         return NULL;
3451 }
3452
3453 /**
3454  * Release flow in RX_CP_TBL.
3455  *
3456  * @param dev
3457  *   Pointer to Ethernet device.
3458  * @flow
3459  *   Parent flow for wich copying is provided.
3460  */
3461 static void
3462 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3463                           struct rte_flow *flow)
3464 {
3465         struct mlx5_flow_mreg_copy_resource *mcp_res;
3466         struct mlx5_priv *priv = dev->data->dev_private;
3467
3468         if (!flow->rix_mreg_copy)
3469                 return;
3470         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3471                                  flow->rix_mreg_copy);
3472         if (!mcp_res || !priv->mreg_cp_tbl)
3473                 return;
3474         if (flow->copy_applied) {
3475                 MLX5_ASSERT(mcp_res->appcnt);
3476                 flow->copy_applied = 0;
3477                 --mcp_res->appcnt;
3478                 if (!mcp_res->appcnt) {
3479                         struct rte_flow *mcp_flow = mlx5_ipool_get
3480                                         (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3481                                         mcp_res->rix_flow);
3482
3483                         if (mcp_flow)
3484                                 flow_drv_remove(dev, mcp_flow);
3485                 }
3486         }
3487         /*
3488          * We do not check availability of metadata registers here,
3489          * because copy resources are not allocated in this case.
3490          */
3491         if (--mcp_res->refcnt)
3492                 return;
3493         MLX5_ASSERT(mcp_res->rix_flow);
3494         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3495         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3496         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3497         flow->rix_mreg_copy = 0;
3498 }
3499
3500 /**
3501  * Start flow in RX_CP_TBL.
3502  *
3503  * @param dev
3504  *   Pointer to Ethernet device.
3505  * @flow
3506  *   Parent flow for wich copying is provided.
3507  *
3508  * @return
3509  *   0 on success, a negative errno value otherwise and rte_errno is set.
3510  */
3511 static int
3512 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3513                             struct rte_flow *flow)
3514 {
3515         struct mlx5_flow_mreg_copy_resource *mcp_res;
3516         struct mlx5_priv *priv = dev->data->dev_private;
3517         int ret;
3518
3519         if (!flow->rix_mreg_copy || flow->copy_applied)
3520                 return 0;
3521         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3522                                  flow->rix_mreg_copy);
3523         if (!mcp_res)
3524                 return 0;
3525         if (!mcp_res->appcnt) {
3526                 struct rte_flow *mcp_flow = mlx5_ipool_get
3527                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3528                                 mcp_res->rix_flow);
3529
3530                 if (mcp_flow) {
3531                         ret = flow_drv_apply(dev, mcp_flow, NULL);
3532                         if (ret)
3533                                 return ret;
3534                 }
3535         }
3536         ++mcp_res->appcnt;
3537         flow->copy_applied = 1;
3538         return 0;
3539 }
3540
3541 /**
3542  * Stop flow in RX_CP_TBL.
3543  *
3544  * @param dev
3545  *   Pointer to Ethernet device.
3546  * @flow
3547  *   Parent flow for wich copying is provided.
3548  */
3549 static void
3550 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3551                            struct rte_flow *flow)
3552 {
3553         struct mlx5_flow_mreg_copy_resource *mcp_res;
3554         struct mlx5_priv *priv = dev->data->dev_private;
3555
3556         if (!flow->rix_mreg_copy || !flow->copy_applied)
3557                 return;
3558         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3559                                  flow->rix_mreg_copy);
3560         if (!mcp_res)
3561                 return;
3562         MLX5_ASSERT(mcp_res->appcnt);
3563         --mcp_res->appcnt;
3564         flow->copy_applied = 0;
3565         if (!mcp_res->appcnt) {
3566                 struct rte_flow *mcp_flow = mlx5_ipool_get
3567                                 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3568                                 mcp_res->rix_flow);
3569
3570                 if (mcp_flow)
3571                         flow_drv_remove(dev, mcp_flow);
3572         }
3573 }
3574
3575 /**
3576  * Remove the default copy action from RX_CP_TBL.
3577  *
3578  * @param dev
3579  *   Pointer to Ethernet device.
3580  */
3581 static void
3582 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3583 {
3584         struct mlx5_flow_mreg_copy_resource *mcp_res;
3585         struct mlx5_priv *priv = dev->data->dev_private;
3586
3587         /* Check if default flow is registered. */
3588         if (!priv->mreg_cp_tbl)
3589                 return;
3590         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3591                                             MLX5_DEFAULT_COPY_ID);
3592         if (!mcp_res)
3593                 return;
3594         MLX5_ASSERT(mcp_res->rix_flow);
3595         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3596         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3597         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3598 }
3599
3600 /**
3601  * Add the default copy action in in RX_CP_TBL.
3602  *
3603  * @param dev
3604  *   Pointer to Ethernet device.
3605  * @param[out] error
3606  *   Perform verbose error reporting if not NULL.
3607  *
3608  * @return
3609  *   0 for success, negative value otherwise and rte_errno is set.
3610  */
3611 static int
3612 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3613                                   struct rte_flow_error *error)
3614 {
3615         struct mlx5_priv *priv = dev->data->dev_private;
3616         struct mlx5_flow_mreg_copy_resource *mcp_res;
3617
3618         /* Check whether extensive metadata feature is engaged. */
3619         if (!priv->config.dv_flow_en ||
3620             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3621             !mlx5_flow_ext_mreg_supported(dev) ||
3622             !priv->sh->dv_regc0_mask)
3623                 return 0;
3624         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3625         if (!mcp_res)
3626                 return -rte_errno;
3627         return 0;
3628 }
3629
3630 /**
3631  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3632  *
3633  * All the flow having Q/RSS action should be split by
3634  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3635  * performs the following,
3636  *   - CQE->flow_tag := reg_c[1] (MARK)
3637  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3638  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3639  * but there should be a flow per each MARK ID set by MARK action.
3640  *
3641  * For the aforementioned reason, if there's a MARK action in flow's action
3642  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3643  * the MARK ID to CQE's flow_tag like,
3644  *   - If reg_c[1] is mark_id,
3645  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3646  *
3647  * For SET_META action which stores value in reg_c[0], as the destination is
3648  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3649  * MARK ID means the default flow. The default flow looks like,
3650  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3651  *
3652  * @param dev
3653  *   Pointer to Ethernet device.
3654  * @param flow
3655  *   Pointer to flow structure.
3656  * @param[in] actions
3657  *   Pointer to the list of actions.
3658  * @param[out] error
3659  *   Perform verbose error reporting if not NULL.
3660  *
3661  * @return
3662  *   0 on success, negative value otherwise and rte_errno is set.
3663  */
3664 static int
3665 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3666                             struct rte_flow *flow,
3667                             const struct rte_flow_action *actions,
3668                             struct rte_flow_error *error)
3669 {
3670         struct mlx5_priv *priv = dev->data->dev_private;
3671         struct mlx5_dev_config *config = &priv->config;
3672         struct mlx5_flow_mreg_copy_resource *mcp_res;
3673         const struct rte_flow_action_mark *mark;
3674
3675         /* Check whether extensive metadata feature is engaged. */
3676         if (!config->dv_flow_en ||
3677             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3678             !mlx5_flow_ext_mreg_supported(dev) ||
3679             !priv->sh->dv_regc0_mask)
3680                 return 0;
3681         /* Find MARK action. */
3682         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3683                 switch (actions->type) {
3684                 case RTE_FLOW_ACTION_TYPE_FLAG:
3685                         mcp_res = flow_mreg_add_copy_action
3686                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3687                         if (!mcp_res)
3688                                 return -rte_errno;
3689                         flow->rix_mreg_copy = mcp_res->idx;
3690                         if (dev->data->dev_started) {
3691                                 mcp_res->appcnt++;
3692                                 flow->copy_applied = 1;
3693                         }
3694                         return 0;
3695                 case RTE_FLOW_ACTION_TYPE_MARK:
3696                         mark = (const struct rte_flow_action_mark *)
3697                                 actions->conf;
3698                         mcp_res =
3699                                 flow_mreg_add_copy_action(dev, mark->id, error);
3700                         if (!mcp_res)
3701                                 return -rte_errno;
3702                         flow->rix_mreg_copy = mcp_res->idx;
3703                         if (dev->data->dev_started) {
3704                                 mcp_res->appcnt++;
3705                                 flow->copy_applied = 1;
3706                         }
3707                         return 0;
3708                 default:
3709                         break;
3710                 }
3711         }
3712         return 0;
3713 }
3714
3715 #define MLX5_MAX_SPLIT_ACTIONS 24
3716 #define MLX5_MAX_SPLIT_ITEMS 24
3717
3718 /**
3719  * Split the hairpin flow.
3720  * Since HW can't support encap and push-vlan on Rx, we move these
3721  * actions to Tx.
3722  * If the count action is after the encap then we also
3723  * move the count action. in this case the count will also measure
3724  * the outer bytes.
3725  *
3726  * @param dev
3727  *   Pointer to Ethernet device.
3728  * @param[in] actions
3729  *   Associated actions (list terminated by the END action).
3730  * @param[out] actions_rx
3731  *   Rx flow actions.
3732  * @param[out] actions_tx
3733  *   Tx flow actions..
3734  * @param[out] pattern_tx
3735  *   The pattern items for the Tx flow.
3736  * @param[out] flow_id
3737  *   The flow ID connected to this flow.
3738  *
3739  * @return
3740  *   0 on success.
3741  */
3742 static int
3743 flow_hairpin_split(struct rte_eth_dev *dev,
3744                    const struct rte_flow_action actions[],
3745                    struct rte_flow_action actions_rx[],
3746                    struct rte_flow_action actions_tx[],
3747                    struct rte_flow_item pattern_tx[],
3748                    uint32_t *flow_id)
3749 {
3750         struct mlx5_priv *priv = dev->data->dev_private;
3751         const struct rte_flow_action_raw_encap *raw_encap;
3752         const struct rte_flow_action_raw_decap *raw_decap;
3753         struct mlx5_rte_flow_action_set_tag *set_tag;
3754         struct rte_flow_action *tag_action;
3755         struct mlx5_rte_flow_item_tag *tag_item;
3756         struct rte_flow_item *item;
3757         char *addr;
3758         int encap = 0;
3759
3760         mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
3761         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3762                 switch (actions->type) {
3763                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3764                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3765                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3766                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3767                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3768                         rte_memcpy(actions_tx, actions,
3769                                sizeof(struct rte_flow_action));
3770                         actions_tx++;
3771                         break;
3772                 case RTE_FLOW_ACTION_TYPE_COUNT:
3773                         if (encap) {
3774                                 rte_memcpy(actions_tx, actions,
3775                                            sizeof(struct rte_flow_action));
3776                                 actions_tx++;
3777                         } else {
3778                                 rte_memcpy(actions_rx, actions,
3779                                            sizeof(struct rte_flow_action));
3780                                 actions_rx++;
3781                         }
3782                         break;
3783                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3784                         raw_encap = actions->conf;
3785                         if (raw_encap->size >
3786                             (sizeof(struct rte_flow_item_eth) +
3787                              sizeof(struct rte_flow_item_ipv4))) {
3788                                 memcpy(actions_tx, actions,
3789                                        sizeof(struct rte_flow_action));
3790                                 actions_tx++;
3791                                 encap = 1;
3792                         } else {
3793                                 rte_memcpy(actions_rx, actions,
3794                                            sizeof(struct rte_flow_action));
3795                                 actions_rx++;
3796                         }
3797                         break;
3798                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3799                         raw_decap = actions->conf;
3800                         if (raw_decap->size <
3801                             (sizeof(struct rte_flow_item_eth) +
3802                              sizeof(struct rte_flow_item_ipv4))) {
3803                                 memcpy(actions_tx, actions,
3804                                        sizeof(struct rte_flow_action));
3805                                 actions_tx++;
3806                         } else {
3807                                 rte_memcpy(actions_rx, actions,
3808                                            sizeof(struct rte_flow_action));
3809                                 actions_rx++;
3810                         }
3811                         break;
3812                 default:
3813                         rte_memcpy(actions_rx, actions,
3814                                    sizeof(struct rte_flow_action));
3815                         actions_rx++;
3816                         break;
3817                 }
3818         }
3819         /* Add set meta action and end action for the Rx flow. */
3820         tag_action = actions_rx;
3821         tag_action->type = (enum rte_flow_action_type)
3822                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3823         actions_rx++;
3824         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
3825         actions_rx++;
3826         set_tag = (void *)actions_rx;
3827         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
3828         MLX5_ASSERT(set_tag->id > REG_NON);
3829         set_tag->data = *flow_id;
3830         tag_action->conf = set_tag;
3831         /* Create Tx item list. */
3832         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
3833         addr = (void *)&pattern_tx[2];
3834         item = pattern_tx;
3835         item->type = (enum rte_flow_item_type)
3836                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3837         tag_item = (void *)addr;
3838         tag_item->data = *flow_id;
3839         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
3840         MLX5_ASSERT(set_tag->id > REG_NON);
3841         item->spec = tag_item;
3842         addr += sizeof(struct mlx5_rte_flow_item_tag);
3843         tag_item = (void *)addr;
3844         tag_item->data = UINT32_MAX;
3845         tag_item->id = UINT16_MAX;
3846         item->mask = tag_item;
3847         item->last = NULL;
3848         item++;
3849         item->type = RTE_FLOW_ITEM_TYPE_END;
3850         return 0;
3851 }
3852
3853 /**
3854  * The last stage of splitting chain, just creates the subflow
3855  * without any modification.
3856  *
3857  * @param[in] dev
3858  *   Pointer to Ethernet device.
3859  * @param[in] flow
3860  *   Parent flow structure pointer.
3861  * @param[in, out] sub_flow
3862  *   Pointer to return the created subflow, may be NULL.
3863  * @param[in] prefix_layers
3864  *   Prefix subflow layers, may be 0.
3865  * @param[in] attr
3866  *   Flow rule attributes.
3867  * @param[in] items
3868  *   Pattern specification (list terminated by the END pattern item).
3869  * @param[in] actions
3870  *   Associated actions (list terminated by the END action).
3871  * @param[in] external
3872  *   This flow rule is created by request external to PMD.
3873  * @param[in] flow_idx
3874  *   This memory pool index to the flow.
3875  * @param[out] error
3876  *   Perform verbose error reporting if not NULL.
3877  * @return
3878  *   0 on success, negative value otherwise
3879  */
3880 static int
3881 flow_create_split_inner(struct rte_eth_dev *dev,
3882                         struct rte_flow *flow,
3883                         struct mlx5_flow **sub_flow,
3884                         uint64_t prefix_layers,
3885                         const struct rte_flow_attr *attr,
3886                         const struct rte_flow_item items[],
3887                         const struct rte_flow_action actions[],
3888                         bool external, uint32_t flow_idx,
3889                         struct rte_flow_error *error)
3890 {
3891         struct mlx5_flow *dev_flow;
3892
3893         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
3894                 flow_idx, error);
3895         if (!dev_flow)
3896                 return -rte_errno;
3897         dev_flow->flow = flow;
3898         dev_flow->external = external;
3899         /* Subflow object was created, we must include one in the list. */
3900         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
3901                       dev_flow->handle, next);
3902         /*
3903          * If dev_flow is as one of the suffix flow, some actions in suffix
3904          * flow may need some user defined item layer flags.
3905          */
3906         if (prefix_layers)
3907                 dev_flow->handle->layers = prefix_layers;
3908         if (sub_flow)
3909                 *sub_flow = dev_flow;
3910         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
3911 }
3912
3913 /**
3914  * Split the meter flow.
3915  *
3916  * As meter flow will split to three sub flow, other than meter
3917  * action, the other actions make sense to only meter accepts
3918  * the packet. If it need to be dropped, no other additional
3919  * actions should be take.
3920  *
3921  * One kind of special action which decapsulates the L3 tunnel
3922  * header will be in the prefix sub flow, as not to take the
3923  * L3 tunnel header into account.
3924  *
3925  * @param dev
3926  *   Pointer to Ethernet device.
3927  * @param[in] items
3928  *   Pattern specification (list terminated by the END pattern item).
3929  * @param[out] sfx_items
3930  *   Suffix flow match items (list terminated by the END pattern item).
3931  * @param[in] actions
3932  *   Associated actions (list terminated by the END action).
3933  * @param[out] actions_sfx
3934  *   Suffix flow actions.
3935  * @param[out] actions_pre
3936  *   Prefix flow actions.
3937  * @param[out] pattern_sfx
3938  *   The pattern items for the suffix flow.
3939  * @param[out] tag_sfx
3940  *   Pointer to suffix flow tag.
3941  *
3942  * @return
3943  *   0 on success.
3944  */
3945 static int
3946 flow_meter_split_prep(struct rte_eth_dev *dev,
3947                  const struct rte_flow_item items[],
3948                  struct rte_flow_item sfx_items[],
3949                  const struct rte_flow_action actions[],
3950                  struct rte_flow_action actions_sfx[],
3951                  struct rte_flow_action actions_pre[])
3952 {
3953         struct rte_flow_action *tag_action = NULL;
3954         struct rte_flow_item *tag_item;
3955         struct mlx5_rte_flow_action_set_tag *set_tag;
3956         struct rte_flow_error error;
3957         const struct rte_flow_action_raw_encap *raw_encap;
3958         const struct rte_flow_action_raw_decap *raw_decap;
3959         struct mlx5_rte_flow_item_tag *tag_spec;
3960         struct mlx5_rte_flow_item_tag *tag_mask;
3961         uint32_t tag_id;
3962         bool copy_vlan = false;
3963
3964         /* Prepare the actions for prefix and suffix flow. */
3965         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3966                 struct rte_flow_action **action_cur = NULL;
3967
3968                 switch (actions->type) {
3969                 case RTE_FLOW_ACTION_TYPE_METER:
3970                         /* Add the extra tag action first. */
3971                         tag_action = actions_pre;
3972                         tag_action->type = (enum rte_flow_action_type)
3973                                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3974                         actions_pre++;
3975                         action_cur = &actions_pre;
3976                         break;
3977                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3978                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3979                         action_cur = &actions_pre;
3980                         break;
3981                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3982                         raw_encap = actions->conf;
3983                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
3984                                 action_cur = &actions_pre;
3985                         break;
3986                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3987                         raw_decap = actions->conf;
3988                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3989                                 action_cur = &actions_pre;
3990                         break;
3991                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3992                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3993                         copy_vlan = true;
3994                         break;
3995                 default:
3996                         break;
3997                 }
3998                 if (!action_cur)
3999                         action_cur = &actions_sfx;
4000                 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
4001                 (*action_cur)++;
4002         }
4003         /* Add end action to the actions. */
4004         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4005         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4006         actions_pre++;
4007         /* Set the tag. */
4008         set_tag = (void *)actions_pre;
4009         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4010         /*
4011          * Get the id from the qrss_pool to make qrss share the id with meter.
4012          */
4013         tag_id = flow_qrss_get_id(dev);
4014         set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
4015         assert(tag_action);
4016         tag_action->conf = set_tag;
4017         /* Prepare the suffix subflow items. */
4018         tag_item = sfx_items++;
4019         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4020                 int item_type = items->type;
4021
4022                 switch (item_type) {
4023                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4024                         memcpy(sfx_items, items, sizeof(*sfx_items));
4025                         sfx_items++;
4026                         break;
4027                 case RTE_FLOW_ITEM_TYPE_VLAN:
4028                         if (copy_vlan) {
4029                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4030                                 /*
4031                                  * Convert to internal match item, it is used
4032                                  * for vlan push and set vid.
4033                                  */
4034                                 sfx_items->type = (enum rte_flow_item_type)
4035                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4036                                 sfx_items++;
4037                         }
4038                         break;
4039                 default:
4040                         break;
4041                 }
4042         }
4043         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4044         sfx_items++;
4045         tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4046         tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
4047         tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
4048         tag_mask = tag_spec + 1;
4049         tag_mask->data = 0xffffff00;
4050         tag_item->type = (enum rte_flow_item_type)
4051                          MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4052         tag_item->spec = tag_spec;
4053         tag_item->last = NULL;
4054         tag_item->mask = tag_mask;
4055         return tag_id;
4056 }
4057
4058 /**
4059  * Split action list having QUEUE/RSS for metadata register copy.
4060  *
4061  * Once Q/RSS action is detected in user's action list, the flow action
4062  * should be split in order to copy metadata registers, which will happen in
4063  * RX_CP_TBL like,
4064  *   - CQE->flow_tag := reg_c[1] (MARK)
4065  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4066  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4067  * This is because the last action of each flow must be a terminal action
4068  * (QUEUE, RSS or DROP).
4069  *
4070  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4071  * stored and kept in the mlx5_flow structure per each sub_flow.
4072  *
4073  * The Q/RSS action is replaced with,
4074  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4075  * And the following JUMP action is added at the end,
4076  *   - JUMP, to RX_CP_TBL.
4077  *
4078  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4079  * flow_create_split_metadata() routine. The flow will look like,
4080  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4081  *
4082  * @param dev
4083  *   Pointer to Ethernet device.
4084  * @param[out] split_actions
4085  *   Pointer to store split actions to jump to CP_TBL.
4086  * @param[in] actions
4087  *   Pointer to the list of original flow actions.
4088  * @param[in] qrss
4089  *   Pointer to the Q/RSS action.
4090  * @param[in] actions_n
4091  *   Number of original actions.
4092  * @param[out] error
4093  *   Perform verbose error reporting if not NULL.
4094  *
4095  * @return
4096  *   non-zero unique flow_id on success, otherwise 0 and
4097  *   error/rte_error are set.
4098  */
4099 static uint32_t
4100 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4101                           struct rte_flow_action *split_actions,
4102                           const struct rte_flow_action *actions,
4103                           const struct rte_flow_action *qrss,
4104                           int actions_n, struct rte_flow_error *error)
4105 {
4106         struct mlx5_rte_flow_action_set_tag *set_tag;
4107         struct rte_flow_action_jump *jump;
4108         const int qrss_idx = qrss - actions;
4109         uint32_t flow_id = 0;
4110         int ret = 0;
4111
4112         /*
4113          * Given actions will be split
4114          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4115          * - Add jump to mreg CP_TBL.
4116          * As a result, there will be one more action.
4117          */
4118         ++actions_n;
4119         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4120         set_tag = (void *)(split_actions + actions_n);
4121         /*
4122          * If tag action is not set to void(it means we are not the meter
4123          * suffix flow), add the tag action. Since meter suffix flow already
4124          * has the tag added.
4125          */
4126         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4127                 /*
4128                  * Allocate the new subflow ID. This one is unique within
4129                  * device and not shared with representors. Otherwise,
4130                  * we would have to resolve multi-thread access synch
4131                  * issue. Each flow on the shared device is appended
4132                  * with source vport identifier, so the resulting
4133                  * flows will be unique in the shared (by master and
4134                  * representors) domain even if they have coinciding
4135                  * IDs.
4136                  */
4137                 flow_id = flow_qrss_get_id(dev);
4138                 if (!flow_id)
4139                         return rte_flow_error_set(error, ENOMEM,
4140                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4141                                                   NULL, "can't allocate id "
4142                                                   "for split Q/RSS subflow");
4143                 /* Internal SET_TAG action to set flow ID. */
4144                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
4145                         .data = flow_id,
4146                 };
4147                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
4148                 if (ret < 0)
4149                         return ret;
4150                 set_tag->id = ret;
4151                 /* Construct new actions array. */
4152                 /* Replace QUEUE/RSS action. */
4153                 split_actions[qrss_idx] = (struct rte_flow_action){
4154                         .type = (enum rte_flow_action_type)
4155                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
4156                         .conf = set_tag,
4157                 };
4158         }
4159         /* JUMP action to jump to mreg copy table (CP_TBL). */
4160         jump = (void *)(set_tag + 1);
4161         *jump = (struct rte_flow_action_jump){
4162                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4163         };
4164         split_actions[actions_n - 2] = (struct rte_flow_action){
4165                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4166                 .conf = jump,
4167         };
4168         split_actions[actions_n - 1] = (struct rte_flow_action){
4169                 .type = RTE_FLOW_ACTION_TYPE_END,
4170         };
4171         return flow_id;
4172 }
4173
4174 /**
4175  * Extend the given action list for Tx metadata copy.
4176  *
4177  * Copy the given action list to the ext_actions and add flow metadata register
4178  * copy action in order to copy reg_a set by WQE to reg_c[0].
4179  *
4180  * @param[out] ext_actions
4181  *   Pointer to the extended action list.
4182  * @param[in] actions
4183  *   Pointer to the list of actions.
4184  * @param[in] actions_n
4185  *   Number of actions in the list.
4186  * @param[out] error
4187  *   Perform verbose error reporting if not NULL.
4188  * @param[in] encap_idx
4189  *   The encap action inndex.
4190  *
4191  * @return
4192  *   0 on success, negative value otherwise
4193  */
4194 static int
4195 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
4196                        struct rte_flow_action *ext_actions,
4197                        const struct rte_flow_action *actions,
4198                        int actions_n, struct rte_flow_error *error,
4199                        int encap_idx)
4200 {
4201         struct mlx5_flow_action_copy_mreg *cp_mreg =
4202                 (struct mlx5_flow_action_copy_mreg *)
4203                         (ext_actions + actions_n + 1);
4204         int ret;
4205
4206         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4207         if (ret < 0)
4208                 return ret;
4209         cp_mreg->dst = ret;
4210         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
4211         if (ret < 0)
4212                 return ret;
4213         cp_mreg->src = ret;
4214         if (encap_idx != 0)
4215                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
4216         if (encap_idx == actions_n - 1) {
4217                 ext_actions[actions_n - 1] = (struct rte_flow_action){
4218                         .type = (enum rte_flow_action_type)
4219                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4220                         .conf = cp_mreg,
4221                 };
4222                 ext_actions[actions_n] = (struct rte_flow_action){
4223                         .type = RTE_FLOW_ACTION_TYPE_END,
4224                 };
4225         } else {
4226                 ext_actions[encap_idx] = (struct rte_flow_action){
4227                         .type = (enum rte_flow_action_type)
4228                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4229                         .conf = cp_mreg,
4230                 };
4231                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
4232                                 sizeof(*ext_actions) * (actions_n - encap_idx));
4233         }
4234         return 0;
4235 }
4236
4237 /**
4238  * The splitting for metadata feature.
4239  *
4240  * - Q/RSS action on NIC Rx should be split in order to pass by
4241  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
4242  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
4243  *
4244  * - All the actions on NIC Tx should have a mreg copy action to
4245  *   copy reg_a from WQE to reg_c[0].
4246  *
4247  * @param dev
4248  *   Pointer to Ethernet device.
4249  * @param[in] flow
4250  *   Parent flow structure pointer.
4251  * @param[in] prefix_layers
4252  *   Prefix flow layer flags.
4253  * @param[in] attr
4254  *   Flow rule attributes.
4255  * @param[in] items
4256  *   Pattern specification (list terminated by the END pattern item).
4257  * @param[in] actions
4258  *   Associated actions (list terminated by the END action).
4259  * @param[in] external
4260  *   This flow rule is created by request external to PMD.
4261  * @param[in] flow_idx
4262  *   This memory pool index to the flow.
4263  * @param[out] error
4264  *   Perform verbose error reporting if not NULL.
4265  * @return
4266  *   0 on success, negative value otherwise
4267  */
4268 static int
4269 flow_create_split_metadata(struct rte_eth_dev *dev,
4270                            struct rte_flow *flow,
4271                            uint64_t prefix_layers,
4272                            const struct rte_flow_attr *attr,
4273                            const struct rte_flow_item items[],
4274                            const struct rte_flow_action actions[],
4275                            bool external, uint32_t flow_idx,
4276                            struct rte_flow_error *error)
4277 {
4278         struct mlx5_priv *priv = dev->data->dev_private;
4279         struct mlx5_dev_config *config = &priv->config;
4280         const struct rte_flow_action *qrss = NULL;
4281         struct rte_flow_action *ext_actions = NULL;
4282         struct mlx5_flow *dev_flow = NULL;
4283         uint32_t qrss_id = 0;
4284         int mtr_sfx = 0;
4285         size_t act_size;
4286         int actions_n;
4287         int encap_idx;
4288         int ret;
4289
4290         /* Check whether extensive metadata feature is engaged. */
4291         if (!config->dv_flow_en ||
4292             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4293             !mlx5_flow_ext_mreg_supported(dev))
4294                 return flow_create_split_inner(dev, flow, NULL, prefix_layers,
4295                                                attr, items, actions, external,
4296                                                flow_idx, error);
4297         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
4298                                                            &encap_idx);
4299         if (qrss) {
4300                 /* Exclude hairpin flows from splitting. */
4301                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
4302                         const struct rte_flow_action_queue *queue;
4303
4304                         queue = qrss->conf;
4305                         if (mlx5_rxq_get_type(dev, queue->index) ==
4306                             MLX5_RXQ_TYPE_HAIRPIN)
4307                                 qrss = NULL;
4308                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
4309                         const struct rte_flow_action_rss *rss;
4310
4311                         rss = qrss->conf;
4312                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
4313                             MLX5_RXQ_TYPE_HAIRPIN)
4314                                 qrss = NULL;
4315                 }
4316         }
4317         if (qrss) {
4318                 /* Check if it is in meter suffix table. */
4319                 mtr_sfx = attr->group == (attr->transfer ?
4320                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4321                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
4322                 /*
4323                  * Q/RSS action on NIC Rx should be split in order to pass by
4324                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
4325                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
4326                  */
4327                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4328                            sizeof(struct rte_flow_action_set_tag) +
4329                            sizeof(struct rte_flow_action_jump);
4330                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4331                                           SOCKET_ID_ANY);
4332                 if (!ext_actions)
4333                         return rte_flow_error_set(error, ENOMEM,
4334                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4335                                                   NULL, "no memory to split "
4336                                                   "metadata flow");
4337                 /*
4338                  * If we are the suffix flow of meter, tag already exist.
4339                  * Set the tag action to void.
4340                  */
4341                 if (mtr_sfx)
4342                         ext_actions[qrss - actions].type =
4343                                                 RTE_FLOW_ACTION_TYPE_VOID;
4344                 else
4345                         ext_actions[qrss - actions].type =
4346                                                 (enum rte_flow_action_type)
4347                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4348                 /*
4349                  * Create the new actions list with removed Q/RSS action
4350                  * and appended set tag and jump to register copy table
4351                  * (RX_CP_TBL). We should preallocate unique tag ID here
4352                  * in advance, because it is needed for set tag action.
4353                  */
4354                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4355                                                     qrss, actions_n, error);
4356                 if (!mtr_sfx && !qrss_id) {
4357                         ret = -rte_errno;
4358                         goto exit;
4359                 }
4360         } else if (attr->egress && !attr->transfer) {
4361                 /*
4362                  * All the actions on NIC Tx should have a metadata register
4363                  * copy action to copy reg_a from WQE to reg_c[meta]
4364                  */
4365                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4366                            sizeof(struct mlx5_flow_action_copy_mreg);
4367                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4368                                           SOCKET_ID_ANY);
4369                 if (!ext_actions)
4370                         return rte_flow_error_set(error, ENOMEM,
4371                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4372                                                   NULL, "no memory to split "
4373                                                   "metadata flow");
4374                 /* Create the action list appended with copy register. */
4375                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4376                                              actions_n, error, encap_idx);
4377                 if (ret < 0)
4378                         goto exit;
4379         }
4380         /* Add the unmodified original or prefix subflow. */
4381         ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr,
4382                                       items, ext_actions ? ext_actions :
4383                                       actions, external, flow_idx, error);
4384         if (ret < 0)
4385                 goto exit;
4386         MLX5_ASSERT(dev_flow);
4387         if (qrss) {
4388                 const struct rte_flow_attr q_attr = {
4389                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4390                         .ingress = 1,
4391                 };
4392                 /* Internal PMD action to set register. */
4393                 struct mlx5_rte_flow_item_tag q_tag_spec = {
4394                         .data = qrss_id,
4395                         .id = REG_NON,
4396                 };
4397                 struct rte_flow_item q_items[] = {
4398                         {
4399                                 .type = (enum rte_flow_item_type)
4400                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4401                                 .spec = &q_tag_spec,
4402                                 .last = NULL,
4403                                 .mask = NULL,
4404                         },
4405                         {
4406                                 .type = RTE_FLOW_ITEM_TYPE_END,
4407                         },
4408                 };
4409                 struct rte_flow_action q_actions[] = {
4410                         {
4411                                 .type = qrss->type,
4412                                 .conf = qrss->conf,
4413                         },
4414                         {
4415                                 .type = RTE_FLOW_ACTION_TYPE_END,
4416                         },
4417                 };
4418                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4419
4420                 /*
4421                  * Configure the tag item only if there is no meter subflow.
4422                  * Since tag is already marked in the meter suffix subflow
4423                  * we can just use the meter suffix items as is.
4424                  */
4425                 if (qrss_id) {
4426                         /* Not meter subflow. */
4427                         MLX5_ASSERT(!mtr_sfx);
4428                         /*
4429                          * Put unique id in prefix flow due to it is destroyed
4430                          * after suffix flow and id will be freed after there
4431                          * is no actual flows with this id and identifier
4432                          * reallocation becomes possible (for example, for
4433                          * other flows in other threads).
4434                          */
4435                         dev_flow->handle->split_flow_id = qrss_id;
4436                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4437                                                    error);
4438                         if (ret < 0)
4439                                 goto exit;
4440                         q_tag_spec.id = ret;
4441                 }
4442                 dev_flow = NULL;
4443                 /* Add suffix subflow to execute Q/RSS. */
4444                 ret = flow_create_split_inner(dev, flow, &dev_flow, layers,
4445                                               &q_attr, mtr_sfx ? items :
4446                                               q_items, q_actions,
4447                                               external, flow_idx, error);
4448                 if (ret < 0)
4449                         goto exit;
4450                 /* qrss ID should be freed if failed. */
4451                 qrss_id = 0;
4452                 MLX5_ASSERT(dev_flow);
4453         }
4454
4455 exit:
4456         /*
4457          * We do not destroy the partially created sub_flows in case of error.
4458          * These ones are included into parent flow list and will be destroyed
4459          * by flow_drv_destroy.
4460          */
4461         flow_qrss_free_id(dev, qrss_id);
4462         mlx5_free(ext_actions);
4463         return ret;
4464 }
4465
4466 /**
4467  * The splitting for meter feature.
4468  *
4469  * - The meter flow will be split to two flows as prefix and
4470  *   suffix flow. The packets make sense only it pass the prefix
4471  *   meter action.
4472  *
4473  * - Reg_C_5 is used for the packet to match betweend prefix and
4474  *   suffix flow.
4475  *
4476  * @param dev
4477  *   Pointer to Ethernet device.
4478  * @param[in] flow
4479  *   Parent flow structure pointer.
4480  * @param[in] attr
4481  *   Flow rule attributes.
4482  * @param[in] items
4483  *   Pattern specification (list terminated by the END pattern item).
4484  * @param[in] actions
4485  *   Associated actions (list terminated by the END action).
4486  * @param[in] external
4487  *   This flow rule is created by request external to PMD.
4488  * @param[in] flow_idx
4489  *   This memory pool index to the flow.
4490  * @param[out] error
4491  *   Perform verbose error reporting if not NULL.
4492  * @return
4493  *   0 on success, negative value otherwise
4494  */
4495 static int
4496 flow_create_split_meter(struct rte_eth_dev *dev,
4497                            struct rte_flow *flow,
4498                            const struct rte_flow_attr *attr,
4499                            const struct rte_flow_item items[],
4500                            const struct rte_flow_action actions[],
4501                            bool external, uint32_t flow_idx,
4502                            struct rte_flow_error *error)
4503 {
4504         struct mlx5_priv *priv = dev->data->dev_private;
4505         struct rte_flow_action *sfx_actions = NULL;
4506         struct rte_flow_action *pre_actions = NULL;
4507         struct rte_flow_item *sfx_items = NULL;
4508         struct mlx5_flow *dev_flow = NULL;
4509         struct rte_flow_attr sfx_attr = *attr;
4510         uint32_t mtr = 0;
4511         uint32_t mtr_tag_id = 0;
4512         size_t act_size;
4513         size_t item_size;
4514         int actions_n = 0;
4515         int ret;
4516
4517         if (priv->mtr_en)
4518                 actions_n = flow_check_meter_action(actions, &mtr);
4519         if (mtr) {
4520                 /* The five prefix actions: meter, decap, encap, tag, end. */
4521                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4522                            sizeof(struct mlx5_rte_flow_action_set_tag);
4523                 /* tag, vlan, port id, end. */
4524 #define METER_SUFFIX_ITEM 4
4525                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4526                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
4527                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4528                                           0, SOCKET_ID_ANY);
4529                 if (!sfx_actions)
4530                         return rte_flow_error_set(error, ENOMEM,
4531                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4532                                                   NULL, "no memory to split "
4533                                                   "meter flow");
4534                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4535                              act_size);
4536                 pre_actions = sfx_actions + actions_n;
4537                 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4538                                                    actions, sfx_actions,
4539                                                    pre_actions);
4540                 if (!mtr_tag_id) {
4541                         ret = -rte_errno;
4542                         goto exit;
4543                 }
4544                 /* Add the prefix subflow. */
4545                 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr,
4546                                               items, pre_actions, external,
4547                                               flow_idx, error);
4548                 if (ret) {
4549                         ret = -rte_errno;
4550                         goto exit;
4551                 }
4552                 dev_flow->handle->split_flow_id = mtr_tag_id;
4553                 /* Setting the sfx group atrr. */
4554                 sfx_attr.group = sfx_attr.transfer ?
4555                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4556                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4557         }
4558         /* Add the prefix subflow. */
4559         ret = flow_create_split_metadata(dev, flow, dev_flow ?
4560                                          flow_get_prefix_layer_flags(dev_flow) :
4561                                          0, &sfx_attr,
4562                                          sfx_items ? sfx_items : items,
4563                                          sfx_actions ? sfx_actions : actions,
4564                                          external, flow_idx, error);
4565 exit:
4566         if (sfx_actions)
4567                 mlx5_free(sfx_actions);
4568         return ret;
4569 }
4570
4571 /**
4572  * Split the flow to subflow set. The splitters might be linked
4573  * in the chain, like this:
4574  * flow_create_split_outer() calls:
4575  *   flow_create_split_meter() calls:
4576  *     flow_create_split_metadata(meter_subflow_0) calls:
4577  *       flow_create_split_inner(metadata_subflow_0)
4578  *       flow_create_split_inner(metadata_subflow_1)
4579  *       flow_create_split_inner(metadata_subflow_2)
4580  *     flow_create_split_metadata(meter_subflow_1) calls:
4581  *       flow_create_split_inner(metadata_subflow_0)
4582  *       flow_create_split_inner(metadata_subflow_1)
4583  *       flow_create_split_inner(metadata_subflow_2)
4584  *
4585  * This provide flexible way to add new levels of flow splitting.
4586  * The all of successfully created subflows are included to the
4587  * parent flow dev_flow list.
4588  *
4589  * @param dev
4590  *   Pointer to Ethernet device.
4591  * @param[in] flow
4592  *   Parent flow structure pointer.
4593  * @param[in] attr
4594  *   Flow rule attributes.
4595  * @param[in] items
4596  *   Pattern specification (list terminated by the END pattern item).
4597  * @param[in] actions
4598  *   Associated actions (list terminated by the END action).
4599  * @param[in] external
4600  *   This flow rule is created by request external to PMD.
4601  * @param[in] flow_idx
4602  *   This memory pool index to the flow.
4603  * @param[out] error
4604  *   Perform verbose error reporting if not NULL.
4605  * @return
4606  *   0 on success, negative value otherwise
4607  */
4608 static int
4609 flow_create_split_outer(struct rte_eth_dev *dev,
4610                         struct rte_flow *flow,
4611                         const struct rte_flow_attr *attr,
4612                         const struct rte_flow_item items[],
4613                         const struct rte_flow_action actions[],
4614                         bool external, uint32_t flow_idx,
4615                         struct rte_flow_error *error)
4616 {
4617         int ret;
4618
4619         ret = flow_create_split_meter(dev, flow, attr, items,
4620                                          actions, external, flow_idx, error);
4621         MLX5_ASSERT(ret <= 0);
4622         return ret;
4623 }
4624
4625 /**
4626  * Create a flow and add it to @p list.
4627  *
4628  * @param dev
4629  *   Pointer to Ethernet device.
4630  * @param list
4631  *   Pointer to a TAILQ flow list. If this parameter NULL,
4632  *   no list insertion occurred, flow is just created,
4633  *   this is caller's responsibility to track the
4634  *   created flow.
4635  * @param[in] attr
4636  *   Flow rule attributes.
4637  * @param[in] items
4638  *   Pattern specification (list terminated by the END pattern item).
4639  * @param[in] actions
4640  *   Associated actions (list terminated by the END action).
4641  * @param[in] external
4642  *   This flow rule is created by request external to PMD.
4643  * @param[out] error
4644  *   Perform verbose error reporting if not NULL.
4645  *
4646  * @return
4647  *   A flow index on success, 0 otherwise and rte_errno is set.
4648  */
4649 static uint32_t
4650 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
4651                  const struct rte_flow_attr *attr,
4652                  const struct rte_flow_item items[],
4653                  const struct rte_flow_action actions[],
4654                  bool external, struct rte_flow_error *error)
4655 {
4656         struct mlx5_priv *priv = dev->data->dev_private;
4657         struct rte_flow *flow = NULL;
4658         struct mlx5_flow *dev_flow;
4659         const struct rte_flow_action_rss *rss;
4660         union {
4661                 struct mlx5_flow_expand_rss buf;
4662                 uint8_t buffer[2048];
4663         } expand_buffer;
4664         union {
4665                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4666                 uint8_t buffer[2048];
4667         } actions_rx;
4668         union {
4669                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4670                 uint8_t buffer[2048];
4671         } actions_hairpin_tx;
4672         union {
4673                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
4674                 uint8_t buffer[2048];
4675         } items_tx;
4676         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
4677         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
4678                                               priv->rss_desc)[!!priv->flow_idx];
4679         const struct rte_flow_action *p_actions_rx = actions;
4680         uint32_t i;
4681         uint32_t idx = 0;
4682         int hairpin_flow;
4683         uint32_t hairpin_id = 0;
4684         struct rte_flow_attr attr_tx = { .priority = 0 };
4685         int ret;
4686
4687         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4688         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
4689                                 external, hairpin_flow, error);
4690         if (ret < 0)
4691                 return 0;
4692         if (hairpin_flow > 0) {
4693                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
4694                         rte_errno = EINVAL;
4695                         return 0;
4696                 }
4697                 flow_hairpin_split(dev, actions, actions_rx.actions,
4698                                    actions_hairpin_tx.actions, items_tx.items,
4699                                    &hairpin_id);
4700                 p_actions_rx = actions_rx.actions;
4701         }
4702         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
4703         if (!flow) {
4704                 rte_errno = ENOMEM;
4705                 goto error_before_flow;
4706         }
4707         flow->drv_type = flow_get_drv_type(dev, attr);
4708         if (hairpin_id != 0)
4709                 flow->hairpin_flow_id = hairpin_id;
4710         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
4711                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
4712         memset(rss_desc, 0, sizeof(*rss_desc));
4713         rss = flow_get_rss_action(p_actions_rx);
4714         if (rss) {
4715                 /*
4716                  * The following information is required by
4717                  * mlx5_flow_hashfields_adjust() in advance.
4718                  */
4719                 rss_desc->level = rss->level;
4720                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
4721                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
4722         }
4723         flow->dev_handles = 0;
4724         if (rss && rss->types) {
4725                 unsigned int graph_root;
4726
4727                 graph_root = find_graph_root(items, rss->level);
4728                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
4729                                            items, rss->types,
4730                                            mlx5_support_expansion, graph_root);
4731                 MLX5_ASSERT(ret > 0 &&
4732                        (unsigned int)ret < sizeof(expand_buffer.buffer));
4733         } else {
4734                 buf->entries = 1;
4735                 buf->entry[0].pattern = (void *)(uintptr_t)items;
4736         }
4737         /*
4738          * Record the start index when there is a nested call. All sub-flows
4739          * need to be translated before another calling.
4740          * No need to use ping-pong buffer to save memory here.
4741          */
4742         if (priv->flow_idx) {
4743                 MLX5_ASSERT(!priv->flow_nested_idx);
4744                 priv->flow_nested_idx = priv->flow_idx;
4745         }
4746         for (i = 0; i < buf->entries; ++i) {
4747                 /*
4748                  * The splitter may create multiple dev_flows,
4749                  * depending on configuration. In the simplest
4750                  * case it just creates unmodified original flow.
4751                  */
4752                 ret = flow_create_split_outer(dev, flow, attr,
4753                                               buf->entry[i].pattern,
4754                                               p_actions_rx, external, idx,
4755                                               error);
4756                 if (ret < 0)
4757                         goto error;
4758         }
4759         /* Create the tx flow. */
4760         if (hairpin_flow) {
4761                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
4762                 attr_tx.ingress = 0;
4763                 attr_tx.egress = 1;
4764                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
4765                                          actions_hairpin_tx.actions,
4766                                          idx, error);
4767                 if (!dev_flow)
4768                         goto error;
4769                 dev_flow->flow = flow;
4770                 dev_flow->external = 0;
4771                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4772                               dev_flow->handle, next);
4773                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
4774                                          items_tx.items,
4775                                          actions_hairpin_tx.actions, error);
4776                 if (ret < 0)
4777                         goto error;
4778         }
4779         /*
4780          * Update the metadata register copy table. If extensive
4781          * metadata feature is enabled and registers are supported
4782          * we might create the extra rte_flow for each unique
4783          * MARK/FLAG action ID.
4784          *
4785          * The table is updated for ingress Flows only, because
4786          * the egress Flows belong to the different device and
4787          * copy table should be updated in peer NIC Rx domain.
4788          */
4789         if (attr->ingress &&
4790             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
4791                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
4792                 if (ret)
4793                         goto error;
4794         }
4795         /*
4796          * If the flow is external (from application) OR device is started, then
4797          * the flow will be applied immediately.
4798          */
4799         if (external || dev->data->dev_started) {
4800                 ret = flow_drv_apply(dev, flow, error);
4801                 if (ret < 0)
4802                         goto error;
4803         }
4804         if (list)
4805                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
4806                              flow, next);
4807         flow_rxq_flags_set(dev, flow);
4808         /* Nested flow creation index recovery. */
4809         priv->flow_idx = priv->flow_nested_idx;
4810         if (priv->flow_nested_idx)
4811                 priv->flow_nested_idx = 0;
4812         return idx;
4813 error:
4814         MLX5_ASSERT(flow);
4815         ret = rte_errno; /* Save rte_errno before cleanup. */
4816         flow_mreg_del_copy_action(dev, flow);
4817         flow_drv_destroy(dev, flow);
4818         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
4819         rte_errno = ret; /* Restore rte_errno. */
4820 error_before_flow:
4821         ret = rte_errno;
4822         if (hairpin_id)
4823                 mlx5_flow_id_release(priv->sh->flow_id_pool,
4824                                      hairpin_id);
4825         rte_errno = ret;
4826         priv->flow_idx = priv->flow_nested_idx;
4827         if (priv->flow_nested_idx)
4828                 priv->flow_nested_idx = 0;
4829         return 0;
4830 }
4831
4832 /**
4833  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
4834  * incoming packets to table 1.
4835  *
4836  * Other flow rules, requested for group n, will be created in
4837  * e-switch table n+1.
4838  * Jump action to e-switch group n will be created to group n+1.
4839  *
4840  * Used when working in switchdev mode, to utilise advantages of table 1
4841  * and above.
4842  *
4843  * @param dev
4844  *   Pointer to Ethernet device.
4845  *
4846  * @return
4847  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
4848  */
4849 struct rte_flow *
4850 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
4851 {
4852         const struct rte_flow_attr attr = {
4853                 .group = 0,
4854                 .priority = 0,
4855                 .ingress = 1,
4856                 .egress = 0,
4857                 .transfer = 1,
4858         };
4859         const struct rte_flow_item pattern = {
4860                 .type = RTE_FLOW_ITEM_TYPE_END,
4861         };
4862         struct rte_flow_action_jump jump = {
4863                 .group = 1,
4864         };
4865         const struct rte_flow_action actions[] = {
4866                 {
4867                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4868                         .conf = &jump,
4869                 },
4870                 {
4871                         .type = RTE_FLOW_ACTION_TYPE_END,
4872                 },
4873         };
4874         struct mlx5_priv *priv = dev->data->dev_private;
4875         struct rte_flow_error error;
4876
4877         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
4878                                                    &attr, &pattern,
4879                                                    actions, false, &error);
4880 }
4881
4882 /**
4883  * Validate a flow supported by the NIC.
4884  *
4885  * @see rte_flow_validate()
4886  * @see rte_flow_ops
4887  */
4888 int
4889 mlx5_flow_validate(struct rte_eth_dev *dev,
4890                    const struct rte_flow_attr *attr,
4891                    const struct rte_flow_item items[],
4892                    const struct rte_flow_action actions[],
4893                    struct rte_flow_error *error)
4894 {
4895         int hairpin_flow;
4896
4897         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4898         return flow_drv_validate(dev, attr, items, actions,
4899                                 true, hairpin_flow, error);
4900 }
4901
4902 /**
4903  * Create a flow.
4904  *
4905  * @see rte_flow_create()
4906  * @see rte_flow_ops
4907  */
4908 struct rte_flow *
4909 mlx5_flow_create(struct rte_eth_dev *dev,
4910                  const struct rte_flow_attr *attr,
4911                  const struct rte_flow_item items[],
4912                  const struct rte_flow_action actions[],
4913                  struct rte_flow_error *error)
4914 {
4915         struct mlx5_priv *priv = dev->data->dev_private;
4916
4917         /*
4918          * If the device is not started yet, it is not allowed to created a
4919          * flow from application. PMD default flows and traffic control flows
4920          * are not affected.
4921          */
4922         if (unlikely(!dev->data->dev_started)) {
4923                 DRV_LOG(DEBUG, "port %u is not started when "
4924                         "inserting a flow", dev->data->port_id);
4925                 rte_flow_error_set(error, ENODEV,
4926                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4927                                    NULL,
4928                                    "port not started");
4929                 return NULL;
4930         }
4931         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
4932                                   attr, items, actions, true, error);
4933 }
4934
4935 /**
4936  * Destroy a flow in a list.
4937  *
4938  * @param dev
4939  *   Pointer to Ethernet device.
4940  * @param list
4941  *   Pointer to the Indexed flow list. If this parameter NULL,
4942  *   there is no flow removal from the list. Be noted that as
4943  *   flow is add to the indexed list, memory of the indexed
4944  *   list points to maybe changed as flow destroyed.
4945  * @param[in] flow_idx
4946  *   Index of flow to destroy.
4947  */
4948 static void
4949 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
4950                   uint32_t flow_idx)
4951 {
4952         struct mlx5_priv *priv = dev->data->dev_private;
4953         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
4954         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
4955                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
4956
4957         if (!flow)
4958                 return;
4959         /*
4960          * Update RX queue flags only if port is started, otherwise it is
4961          * already clean.
4962          */
4963         if (dev->data->dev_started)
4964                 flow_rxq_flags_trim(dev, flow);
4965         if (flow->hairpin_flow_id)
4966                 mlx5_flow_id_release(priv->sh->flow_id_pool,
4967                                      flow->hairpin_flow_id);
4968         flow_drv_destroy(dev, flow);
4969         if (list)
4970                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
4971                              flow_idx, flow, next);
4972         flow_mreg_del_copy_action(dev, flow);
4973         if (flow->fdir) {
4974                 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
4975                         if (priv_fdir_flow->rix_flow == flow_idx)
4976                                 break;
4977                 }
4978                 if (priv_fdir_flow) {
4979                         LIST_REMOVE(priv_fdir_flow, next);
4980                         mlx5_free(priv_fdir_flow->fdir);
4981                         mlx5_free(priv_fdir_flow);
4982                 }
4983         }
4984         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
4985 }
4986
4987 /**
4988  * Destroy all flows.
4989  *
4990  * @param dev
4991  *   Pointer to Ethernet device.
4992  * @param list
4993  *   Pointer to the Indexed flow list.
4994  * @param active
4995  *   If flushing is called avtively.
4996  */
4997 void
4998 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
4999 {
5000         uint32_t num_flushed = 0;
5001
5002         while (*list) {
5003                 flow_list_destroy(dev, list, *list);
5004                 num_flushed++;
5005         }
5006         if (active) {
5007                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
5008                         dev->data->port_id, num_flushed);
5009         }
5010 }
5011
5012 /**
5013  * Remove all flows.
5014  *
5015  * @param dev
5016  *   Pointer to Ethernet device.
5017  * @param list
5018  *   Pointer to the Indexed flow list.
5019  */
5020 void
5021 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
5022 {
5023         struct mlx5_priv *priv = dev->data->dev_private;
5024         struct rte_flow *flow = NULL;
5025         uint32_t idx;
5026
5027         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
5028                       flow, next) {
5029                 flow_drv_remove(dev, flow);
5030                 flow_mreg_stop_copy_action(dev, flow);
5031         }
5032         flow_mreg_del_default_copy_action(dev);
5033         flow_rxq_flags_clear(dev);
5034 }
5035
5036 /**
5037  * Add all flows.
5038  *
5039  * @param dev
5040  *   Pointer to Ethernet device.
5041  * @param list
5042  *   Pointer to the Indexed flow list.
5043  *
5044  * @return
5045  *   0 on success, a negative errno value otherwise and rte_errno is set.
5046  */
5047 int
5048 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
5049 {
5050         struct mlx5_priv *priv = dev->data->dev_private;
5051         struct rte_flow *flow = NULL;
5052         struct rte_flow_error error;
5053         uint32_t idx;
5054         int ret = 0;
5055
5056         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5057         ret = flow_mreg_add_default_copy_action(dev, &error);
5058         if (ret < 0)
5059                 return -rte_errno;
5060         /* Apply Flows created by application. */
5061         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
5062                       flow, next) {
5063                 ret = flow_mreg_start_copy_action(dev, flow);
5064                 if (ret < 0)
5065                         goto error;
5066                 ret = flow_drv_apply(dev, flow, &error);
5067                 if (ret < 0)
5068                         goto error;
5069                 flow_rxq_flags_set(dev, flow);
5070         }
5071         return 0;
5072 error:
5073         ret = rte_errno; /* Save rte_errno before cleanup. */
5074         mlx5_flow_stop(dev, list);
5075         rte_errno = ret; /* Restore rte_errno. */
5076         return -rte_errno;
5077 }
5078
5079 /**
5080  * Stop all default actions for flows.
5081  *
5082  * @param dev
5083  *   Pointer to Ethernet device.
5084  */
5085 void
5086 mlx5_flow_stop_default(struct rte_eth_dev *dev)
5087 {
5088         flow_mreg_del_default_copy_action(dev);
5089         flow_rxq_flags_clear(dev);
5090 }
5091
5092 /**
5093  * Start all default actions for flows.
5094  *
5095  * @param dev
5096  *   Pointer to Ethernet device.
5097  * @return
5098  *   0 on success, a negative errno value otherwise and rte_errno is set.
5099  */
5100 int
5101 mlx5_flow_start_default(struct rte_eth_dev *dev)
5102 {
5103         struct rte_flow_error error;
5104
5105         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
5106         return flow_mreg_add_default_copy_action(dev, &error);
5107 }
5108
5109 /**
5110  * Allocate intermediate resources for flow creation.
5111  *
5112  * @param dev
5113  *   Pointer to Ethernet device.
5114  */
5115 void
5116 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
5117 {
5118         struct mlx5_priv *priv = dev->data->dev_private;
5119
5120         if (!priv->inter_flows) {
5121                 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO,
5122                                     MLX5_NUM_MAX_DEV_FLOWS *
5123                                     sizeof(struct mlx5_flow) +
5124                                     (sizeof(struct mlx5_flow_rss_desc) +
5125                                     sizeof(uint16_t) * UINT16_MAX) * 2, 0,
5126                                     SOCKET_ID_ANY);
5127                 if (!priv->inter_flows) {
5128                         DRV_LOG(ERR, "can't allocate intermediate memory.");
5129                         return;
5130                 }
5131         }
5132         priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
5133                          [MLX5_NUM_MAX_DEV_FLOWS];
5134         /* Reset the index. */
5135         priv->flow_idx = 0;
5136         priv->flow_nested_idx = 0;
5137 }
5138
5139 /**
5140  * Free intermediate resources for flows.
5141  *
5142  * @param dev
5143  *   Pointer to Ethernet device.
5144  */
5145 void
5146 mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
5147 {
5148         struct mlx5_priv *priv = dev->data->dev_private;
5149
5150         mlx5_free(priv->inter_flows);
5151         priv->inter_flows = NULL;
5152 }
5153
5154 /**
5155  * Verify the flow list is empty
5156  *
5157  * @param dev
5158  *  Pointer to Ethernet device.
5159  *
5160  * @return the number of flows not released.
5161  */
5162 int
5163 mlx5_flow_verify(struct rte_eth_dev *dev)
5164 {
5165         struct mlx5_priv *priv = dev->data->dev_private;
5166         struct rte_flow *flow;
5167         uint32_t idx;
5168         int ret = 0;
5169
5170         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
5171                       flow, next) {
5172                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
5173                         dev->data->port_id, (void *)flow);
5174                 ++ret;
5175         }
5176         return ret;
5177 }
5178
5179 /**
5180  * Enable default hairpin egress flow.
5181  *
5182  * @param dev
5183  *   Pointer to Ethernet device.
5184  * @param queue
5185  *   The queue index.
5186  *
5187  * @return
5188  *   0 on success, a negative errno value otherwise and rte_errno is set.
5189  */
5190 int
5191 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
5192                             uint32_t queue)
5193 {
5194         struct mlx5_priv *priv = dev->data->dev_private;
5195         const struct rte_flow_attr attr = {
5196                 .egress = 1,
5197                 .priority = 0,
5198         };
5199         struct mlx5_rte_flow_item_tx_queue queue_spec = {
5200                 .queue = queue,
5201         };
5202         struct mlx5_rte_flow_item_tx_queue queue_mask = {
5203                 .queue = UINT32_MAX,
5204         };
5205         struct rte_flow_item items[] = {
5206                 {
5207                         .type = (enum rte_flow_item_type)
5208                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
5209                         .spec = &queue_spec,
5210                         .last = NULL,
5211                         .mask = &queue_mask,
5212                 },
5213                 {
5214                         .type = RTE_FLOW_ITEM_TYPE_END,
5215                 },
5216         };
5217         struct rte_flow_action_jump jump = {
5218                 .group = MLX5_HAIRPIN_TX_TABLE,
5219         };
5220         struct rte_flow_action actions[2];
5221         uint32_t flow_idx;
5222         struct rte_flow_error error;
5223
5224         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
5225         actions[0].conf = &jump;
5226         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
5227         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5228                                 &attr, items, actions, false, &error);
5229         if (!flow_idx) {
5230                 DRV_LOG(DEBUG,
5231                         "Failed to create ctrl flow: rte_errno(%d),"
5232                         " type(%d), message(%s)",
5233                         rte_errno, error.type,
5234                         error.message ? error.message : " (no stated reason)");
5235                 return -rte_errno;
5236         }
5237         return 0;
5238 }
5239
5240 /**
5241  * Enable a control flow configured from the control plane.
5242  *
5243  * @param dev
5244  *   Pointer to Ethernet device.
5245  * @param eth_spec
5246  *   An Ethernet flow spec to apply.
5247  * @param eth_mask
5248  *   An Ethernet flow mask to apply.
5249  * @param vlan_spec
5250  *   A VLAN flow spec to apply.
5251  * @param vlan_mask
5252  *   A VLAN flow mask to apply.
5253  *
5254  * @return
5255  *   0 on success, a negative errno value otherwise and rte_errno is set.
5256  */
5257 int
5258 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
5259                     struct rte_flow_item_eth *eth_spec,
5260                     struct rte_flow_item_eth *eth_mask,
5261                     struct rte_flow_item_vlan *vlan_spec,
5262                     struct rte_flow_item_vlan *vlan_mask)
5263 {
5264         struct mlx5_priv *priv = dev->data->dev_private;
5265         const struct rte_flow_attr attr = {
5266                 .ingress = 1,
5267                 .priority = MLX5_FLOW_PRIO_RSVD,
5268         };
5269         struct rte_flow_item items[] = {
5270                 {
5271                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5272                         .spec = eth_spec,
5273                         .last = NULL,
5274                         .mask = eth_mask,
5275                 },
5276                 {
5277                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
5278                                               RTE_FLOW_ITEM_TYPE_END,
5279                         .spec = vlan_spec,
5280                         .last = NULL,
5281                         .mask = vlan_mask,
5282                 },
5283                 {
5284                         .type = RTE_FLOW_ITEM_TYPE_END,
5285                 },
5286         };
5287         uint16_t queue[priv->reta_idx_n];
5288         struct rte_flow_action_rss action_rss = {
5289                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
5290                 .level = 0,
5291                 .types = priv->rss_conf.rss_hf,
5292                 .key_len = priv->rss_conf.rss_key_len,
5293                 .queue_num = priv->reta_idx_n,
5294                 .key = priv->rss_conf.rss_key,
5295                 .queue = queue,
5296         };
5297         struct rte_flow_action actions[] = {
5298                 {
5299                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5300                         .conf = &action_rss,
5301                 },
5302                 {
5303                         .type = RTE_FLOW_ACTION_TYPE_END,
5304                 },
5305         };
5306         uint32_t flow_idx;
5307         struct rte_flow_error error;
5308         unsigned int i;
5309
5310         if (!priv->reta_idx_n || !priv->rxqs_n) {
5311                 return 0;
5312         }
5313         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
5314                 action_rss.types = 0;
5315         for (i = 0; i != priv->reta_idx_n; ++i)
5316                 queue[i] = (*priv->reta_idx)[i];
5317         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5318                                 &attr, items, actions, false, &error);
5319         if (!flow_idx)
5320                 return -rte_errno;
5321         return 0;
5322 }
5323
5324 /**
5325  * Enable a flow control configured from the control plane.
5326  *
5327  * @param dev
5328  *   Pointer to Ethernet device.
5329  * @param eth_spec
5330  *   An Ethernet flow spec to apply.
5331  * @param eth_mask
5332  *   An Ethernet flow mask to apply.
5333  *
5334  * @return
5335  *   0 on success, a negative errno value otherwise and rte_errno is set.
5336  */
5337 int
5338 mlx5_ctrl_flow(struct rte_eth_dev *dev,
5339                struct rte_flow_item_eth *eth_spec,
5340                struct rte_flow_item_eth *eth_mask)
5341 {
5342         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
5343 }
5344
5345 /**
5346  * Create default miss flow rule matching lacp traffic
5347  *
5348  * @param dev
5349  *   Pointer to Ethernet device.
5350  * @param eth_spec
5351  *   An Ethernet flow spec to apply.
5352  *
5353  * @return
5354  *   0 on success, a negative errno value otherwise and rte_errno is set.
5355  */
5356 int
5357 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
5358 {
5359         struct mlx5_priv *priv = dev->data->dev_private;
5360         /*
5361          * The LACP matching is done by only using ether type since using
5362          * a multicast dst mac causes kernel to give low priority to this flow.
5363          */
5364         static const struct rte_flow_item_eth lacp_spec = {
5365                 .type = RTE_BE16(0x8809),
5366         };
5367         static const struct rte_flow_item_eth lacp_mask = {
5368                 .type = 0xffff,
5369         };
5370         const struct rte_flow_attr attr = {
5371                 .ingress = 1,
5372         };
5373         struct rte_flow_item items[] = {
5374                 {
5375                         .type = RTE_FLOW_ITEM_TYPE_ETH,
5376                         .spec = &lacp_spec,
5377                         .mask = &lacp_mask,
5378                 },
5379                 {
5380                         .type = RTE_FLOW_ITEM_TYPE_END,
5381                 },
5382         };
5383         struct rte_flow_action actions[] = {
5384                 {
5385                         .type = (enum rte_flow_action_type)
5386                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
5387                 },
5388                 {
5389                         .type = RTE_FLOW_ACTION_TYPE_END,
5390                 },
5391         };
5392         struct rte_flow_error error;
5393         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5394                                 &attr, items, actions, false, &error);
5395
5396         if (!flow_idx)
5397                 return -rte_errno;
5398         return 0;
5399 }
5400
5401 /**
5402  * Destroy a flow.
5403  *
5404  * @see rte_flow_destroy()
5405  * @see rte_flow_ops
5406  */
5407 int
5408 mlx5_flow_destroy(struct rte_eth_dev *dev,
5409                   struct rte_flow *flow,
5410                   struct rte_flow_error *error __rte_unused)
5411 {
5412         struct mlx5_priv *priv = dev->data->dev_private;
5413
5414         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
5415         return 0;
5416 }
5417
5418 /**
5419  * Destroy all flows.
5420  *
5421  * @see rte_flow_flush()
5422  * @see rte_flow_ops
5423  */
5424 int
5425 mlx5_flow_flush(struct rte_eth_dev *dev,
5426                 struct rte_flow_error *error __rte_unused)
5427 {
5428         struct mlx5_priv *priv = dev->data->dev_private;
5429
5430         mlx5_flow_list_flush(dev, &priv->flows, false);
5431         return 0;
5432 }
5433
5434 /**
5435  * Isolated mode.
5436  *
5437  * @see rte_flow_isolate()
5438  * @see rte_flow_ops
5439  */
5440 int
5441 mlx5_flow_isolate(struct rte_eth_dev *dev,
5442                   int enable,
5443                   struct rte_flow_error *error)
5444 {
5445         struct mlx5_priv *priv = dev->data->dev_private;
5446
5447         if (dev->data->dev_started) {
5448                 rte_flow_error_set(error, EBUSY,
5449                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5450                                    NULL,
5451                                    "port must be stopped first");
5452                 return -rte_errno;
5453         }
5454         priv->isolated = !!enable;
5455         if (enable)
5456                 dev->dev_ops = &mlx5_os_dev_ops_isolate;
5457         else
5458                 dev->dev_ops = &mlx5_os_dev_ops;
5459
5460         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
5461         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
5462
5463         return 0;
5464 }
5465
5466 /**
5467  * Query a flow.
5468  *
5469  * @see rte_flow_query()
5470  * @see rte_flow_ops
5471  */
5472 static int
5473 flow_drv_query(struct rte_eth_dev *dev,
5474                uint32_t flow_idx,
5475                const struct rte_flow_action *actions,
5476                void *data,
5477                struct rte_flow_error *error)
5478 {
5479         struct mlx5_priv *priv = dev->data->dev_private;
5480         const struct mlx5_flow_driver_ops *fops;
5481         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5482                                                [MLX5_IPOOL_RTE_FLOW],
5483                                                flow_idx);
5484         enum mlx5_flow_drv_type ftype;
5485
5486         if (!flow) {
5487                 return rte_flow_error_set(error, ENOENT,
5488                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5489                           NULL,
5490                           "invalid flow handle");
5491         }
5492         ftype = flow->drv_type;
5493         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
5494         fops = flow_get_drv_ops(ftype);
5495
5496         return fops->query(dev, flow, actions, data, error);
5497 }
5498
5499 /**
5500  * Query a flow.
5501  *
5502  * @see rte_flow_query()
5503  * @see rte_flow_ops
5504  */
5505 int
5506 mlx5_flow_query(struct rte_eth_dev *dev,
5507                 struct rte_flow *flow,
5508                 const struct rte_flow_action *actions,
5509                 void *data,
5510                 struct rte_flow_error *error)
5511 {
5512         int ret;
5513
5514         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
5515                              error);
5516         if (ret < 0)
5517                 return ret;
5518         return 0;
5519 }
5520
5521 /**
5522  * Convert a flow director filter to a generic flow.
5523  *
5524  * @param dev
5525  *   Pointer to Ethernet device.
5526  * @param fdir_filter
5527  *   Flow director filter to add.
5528  * @param attributes
5529  *   Generic flow parameters structure.
5530  *
5531  * @return
5532  *   0 on success, a negative errno value otherwise and rte_errno is set.
5533  */
5534 static int
5535 flow_fdir_filter_convert(struct rte_eth_dev *dev,
5536                          const struct rte_eth_fdir_filter *fdir_filter,
5537                          struct mlx5_fdir *attributes)
5538 {
5539         struct mlx5_priv *priv = dev->data->dev_private;
5540         const struct rte_eth_fdir_input *input = &fdir_filter->input;
5541         const struct rte_eth_fdir_masks *mask =
5542                 &dev->data->dev_conf.fdir_conf.mask;
5543
5544         /* Validate queue number. */
5545         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
5546                 DRV_LOG(ERR, "port %u invalid queue number %d",
5547                         dev->data->port_id, fdir_filter->action.rx_queue);
5548                 rte_errno = EINVAL;
5549                 return -rte_errno;
5550         }
5551         attributes->attr.ingress = 1;
5552         attributes->items[0] = (struct rte_flow_item) {
5553                 .type = RTE_FLOW_ITEM_TYPE_ETH,
5554                 .spec = &attributes->l2,
5555                 .mask = &attributes->l2_mask,
5556         };
5557         switch (fdir_filter->action.behavior) {
5558         case RTE_ETH_FDIR_ACCEPT:
5559                 attributes->actions[0] = (struct rte_flow_action){
5560                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
5561                         .conf = &attributes->queue,
5562                 };
5563                 break;
5564         case RTE_ETH_FDIR_REJECT:
5565                 attributes->actions[0] = (struct rte_flow_action){
5566                         .type = RTE_FLOW_ACTION_TYPE_DROP,
5567                 };
5568                 break;
5569         default:
5570                 DRV_LOG(ERR, "port %u invalid behavior %d",
5571                         dev->data->port_id,
5572                         fdir_filter->action.behavior);
5573                 rte_errno = ENOTSUP;
5574                 return -rte_errno;
5575         }
5576         attributes->queue.index = fdir_filter->action.rx_queue;
5577         /* Handle L3. */
5578         switch (fdir_filter->input.flow_type) {
5579         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5580         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5581         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5582                 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
5583                         .src_addr = input->flow.ip4_flow.src_ip,
5584                         .dst_addr = input->flow.ip4_flow.dst_ip,
5585                         .time_to_live = input->flow.ip4_flow.ttl,
5586                         .type_of_service = input->flow.ip4_flow.tos,
5587                 };
5588                 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
5589                         .src_addr = mask->ipv4_mask.src_ip,
5590                         .dst_addr = mask->ipv4_mask.dst_ip,
5591                         .time_to_live = mask->ipv4_mask.ttl,
5592                         .type_of_service = mask->ipv4_mask.tos,
5593                         .next_proto_id = mask->ipv4_mask.proto,
5594                 };
5595                 attributes->items[1] = (struct rte_flow_item){
5596                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
5597                         .spec = &attributes->l3,
5598                         .mask = &attributes->l3_mask,
5599                 };
5600                 break;
5601         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5602         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5603         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5604                 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
5605                         .hop_limits = input->flow.ipv6_flow.hop_limits,
5606                         .proto = input->flow.ipv6_flow.proto,
5607                 };
5608
5609                 memcpy(attributes->l3.ipv6.hdr.src_addr,
5610                        input->flow.ipv6_flow.src_ip,
5611                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5612                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
5613                        input->flow.ipv6_flow.dst_ip,
5614                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5615                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
5616                        mask->ipv6_mask.src_ip,
5617                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5618                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
5619                        mask->ipv6_mask.dst_ip,
5620                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5621                 attributes->items[1] = (struct rte_flow_item){
5622                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
5623                         .spec = &attributes->l3,
5624                         .mask = &attributes->l3_mask,
5625                 };
5626                 break;
5627         default:
5628                 DRV_LOG(ERR, "port %u invalid flow type%d",
5629                         dev->data->port_id, fdir_filter->input.flow_type);
5630                 rte_errno = ENOTSUP;
5631                 return -rte_errno;
5632         }
5633         /* Handle L4. */
5634         switch (fdir_filter->input.flow_type) {
5635         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5636                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5637                         .src_port = input->flow.udp4_flow.src_port,
5638                         .dst_port = input->flow.udp4_flow.dst_port,
5639                 };
5640                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5641                         .src_port = mask->src_port_mask,
5642                         .dst_port = mask->dst_port_mask,
5643                 };
5644                 attributes->items[2] = (struct rte_flow_item){
5645                         .type = RTE_FLOW_ITEM_TYPE_UDP,
5646                         .spec = &attributes->l4,
5647                         .mask = &attributes->l4_mask,
5648                 };
5649                 break;
5650         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5651                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5652                         .src_port = input->flow.tcp4_flow.src_port,
5653                         .dst_port = input->flow.tcp4_flow.dst_port,
5654                 };
5655                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5656                         .src_port = mask->src_port_mask,
5657                         .dst_port = mask->dst_port_mask,
5658                 };
5659                 attributes->items[2] = (struct rte_flow_item){
5660                         .type = RTE_FLOW_ITEM_TYPE_TCP,
5661                         .spec = &attributes->l4,
5662                         .mask = &attributes->l4_mask,
5663                 };
5664                 break;
5665         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5666                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5667                         .src_port = input->flow.udp6_flow.src_port,
5668                         .dst_port = input->flow.udp6_flow.dst_port,
5669                 };
5670                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5671                         .src_port = mask->src_port_mask,
5672                         .dst_port = mask->dst_port_mask,
5673                 };
5674                 attributes->items[2] = (struct rte_flow_item){
5675                         .type = RTE_FLOW_ITEM_TYPE_UDP,
5676                         .spec = &attributes->l4,
5677                         .mask = &attributes->l4_mask,
5678                 };
5679                 break;
5680         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5681                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5682                         .src_port = input->flow.tcp6_flow.src_port,
5683                         .dst_port = input->flow.tcp6_flow.dst_port,
5684                 };
5685                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5686                         .src_port = mask->src_port_mask,
5687                         .dst_port = mask->dst_port_mask,
5688                 };
5689                 attributes->items[2] = (struct rte_flow_item){
5690                         .type = RTE_FLOW_ITEM_TYPE_TCP,
5691                         .spec = &attributes->l4,
5692                         .mask = &attributes->l4_mask,
5693                 };
5694                 break;
5695         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5696         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5697                 break;
5698         default:
5699                 DRV_LOG(ERR, "port %u invalid flow type%d",
5700                         dev->data->port_id, fdir_filter->input.flow_type);
5701                 rte_errno = ENOTSUP;
5702                 return -rte_errno;
5703         }
5704         return 0;
5705 }
5706
5707 #define FLOW_FDIR_CMP(f1, f2, fld) \
5708         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
5709
5710 /**
5711  * Compare two FDIR flows. If items and actions are identical, the two flows are
5712  * regarded as same.
5713  *
5714  * @param dev
5715  *   Pointer to Ethernet device.
5716  * @param f1
5717  *   FDIR flow to compare.
5718  * @param f2
5719  *   FDIR flow to compare.
5720  *
5721  * @return
5722  *   Zero on match, 1 otherwise.
5723  */
5724 static int
5725 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
5726 {
5727         if (FLOW_FDIR_CMP(f1, f2, attr) ||
5728             FLOW_FDIR_CMP(f1, f2, l2) ||
5729             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
5730             FLOW_FDIR_CMP(f1, f2, l3) ||
5731             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
5732             FLOW_FDIR_CMP(f1, f2, l4) ||
5733             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
5734             FLOW_FDIR_CMP(f1, f2, actions[0].type))
5735                 return 1;
5736         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
5737             FLOW_FDIR_CMP(f1, f2, queue))
5738                 return 1;
5739         return 0;
5740 }
5741
5742 /**
5743  * Search device flow list to find out a matched FDIR flow.
5744  *
5745  * @param dev
5746  *   Pointer to Ethernet device.
5747  * @param fdir_flow
5748  *   FDIR flow to lookup.
5749  *
5750  * @return
5751  *   Index of flow if found, 0 otherwise.
5752  */
5753 static uint32_t
5754 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
5755 {
5756         struct mlx5_priv *priv = dev->data->dev_private;
5757         uint32_t flow_idx = 0;
5758         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5759
5760         MLX5_ASSERT(fdir_flow);
5761         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5762                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
5763                         DRV_LOG(DEBUG, "port %u found FDIR flow %u",
5764                                 dev->data->port_id, flow_idx);
5765                         flow_idx = priv_fdir_flow->rix_flow;
5766                         break;
5767                 }
5768         }
5769         return flow_idx;
5770 }
5771
5772 /**
5773  * Add new flow director filter and store it in list.
5774  *
5775  * @param dev
5776  *   Pointer to Ethernet device.
5777  * @param fdir_filter
5778  *   Flow director filter to add.
5779  *
5780  * @return
5781  *   0 on success, a negative errno value otherwise and rte_errno is set.
5782  */
5783 static int
5784 flow_fdir_filter_add(struct rte_eth_dev *dev,
5785                      const struct rte_eth_fdir_filter *fdir_filter)
5786 {
5787         struct mlx5_priv *priv = dev->data->dev_private;
5788         struct mlx5_fdir *fdir_flow;
5789         struct rte_flow *flow;
5790         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5791         uint32_t flow_idx;
5792         int ret;
5793
5794         fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0,
5795                                 SOCKET_ID_ANY);
5796         if (!fdir_flow) {
5797                 rte_errno = ENOMEM;
5798                 return -rte_errno;
5799         }
5800         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
5801         if (ret)
5802                 goto error;
5803         flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
5804         if (flow_idx) {
5805                 rte_errno = EEXIST;
5806                 goto error;
5807         }
5808         priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO,
5809                                      sizeof(struct mlx5_fdir_flow),
5810                                      0, SOCKET_ID_ANY);
5811         if (!priv_fdir_flow) {
5812                 rte_errno = ENOMEM;
5813                 goto error;
5814         }
5815         flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
5816                                     fdir_flow->items, fdir_flow->actions, true,
5817                                     NULL);
5818         flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5819         if (!flow)
5820                 goto error;
5821         flow->fdir = 1;
5822         priv_fdir_flow->fdir = fdir_flow;
5823         priv_fdir_flow->rix_flow = flow_idx;
5824         LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
5825         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
5826                 dev->data->port_id, (void *)flow);
5827         return 0;
5828 error:
5829         mlx5_free(priv_fdir_flow);
5830         mlx5_free(fdir_flow);
5831         return -rte_errno;
5832 }
5833
5834 /**
5835  * Delete specific filter.
5836  *
5837  * @param dev
5838  *   Pointer to Ethernet device.
5839  * @param fdir_filter
5840  *   Filter to be deleted.
5841  *
5842  * @return
5843  *   0 on success, a negative errno value otherwise and rte_errno is set.
5844  */
5845 static int
5846 flow_fdir_filter_delete(struct rte_eth_dev *dev,
5847                         const struct rte_eth_fdir_filter *fdir_filter)
5848 {
5849         struct mlx5_priv *priv = dev->data->dev_private;
5850         uint32_t flow_idx;
5851         struct mlx5_fdir fdir_flow = {
5852                 .attr.group = 0,
5853         };
5854         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5855         int ret;
5856
5857         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
5858         if (ret)
5859                 return -rte_errno;
5860         LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5861                 /* Find the fdir in priv list */
5862                 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
5863                         break;
5864         }
5865         if (!priv_fdir_flow)
5866                 return 0;
5867         LIST_REMOVE(priv_fdir_flow, next);
5868         flow_idx = priv_fdir_flow->rix_flow;
5869         flow_list_destroy(dev, &priv->flows, flow_idx);
5870         mlx5_free(priv_fdir_flow->fdir);
5871         mlx5_free(priv_fdir_flow);
5872         DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
5873                 dev->data->port_id, flow_idx);
5874         return 0;
5875 }
5876
5877 /**
5878  * Update queue for specific filter.
5879  *
5880  * @param dev
5881  *   Pointer to Ethernet device.
5882  * @param fdir_filter
5883  *   Filter to be updated.
5884  *
5885  * @return
5886  *   0 on success, a negative errno value otherwise and rte_errno is set.
5887  */
5888 static int
5889 flow_fdir_filter_update(struct rte_eth_dev *dev,
5890                         const struct rte_eth_fdir_filter *fdir_filter)
5891 {
5892         int ret;
5893
5894         ret = flow_fdir_filter_delete(dev, fdir_filter);
5895         if (ret)
5896                 return ret;
5897         return flow_fdir_filter_add(dev, fdir_filter);
5898 }
5899
5900 /**
5901  * Flush all filters.
5902  *
5903  * @param dev
5904  *   Pointer to Ethernet device.
5905  */
5906 static void
5907 flow_fdir_filter_flush(struct rte_eth_dev *dev)
5908 {
5909         struct mlx5_priv *priv = dev->data->dev_private;
5910         struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5911
5912         while (!LIST_EMPTY(&priv->fdir_flows)) {
5913                 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
5914                 LIST_REMOVE(priv_fdir_flow, next);
5915                 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
5916                 mlx5_free(priv_fdir_flow->fdir);
5917                 mlx5_free(priv_fdir_flow);
5918         }
5919 }
5920
5921 /**
5922  * Get flow director information.
5923  *
5924  * @param dev
5925  *   Pointer to Ethernet device.
5926  * @param[out] fdir_info
5927  *   Resulting flow director information.
5928  */
5929 static void
5930 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
5931 {
5932         struct rte_eth_fdir_masks *mask =
5933                 &dev->data->dev_conf.fdir_conf.mask;
5934
5935         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
5936         fdir_info->guarant_spc = 0;
5937         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
5938         fdir_info->max_flexpayload = 0;
5939         fdir_info->flow_types_mask[0] = 0;
5940         fdir_info->flex_payload_unit = 0;
5941         fdir_info->max_flex_payload_segment_num = 0;
5942         fdir_info->flex_payload_limit = 0;
5943         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
5944 }
5945
5946 /**
5947  * Deal with flow director operations.
5948  *
5949  * @param dev
5950  *   Pointer to Ethernet device.
5951  * @param filter_op
5952  *   Operation to perform.
5953  * @param arg
5954  *   Pointer to operation-specific structure.
5955  *
5956  * @return
5957  *   0 on success, a negative errno value otherwise and rte_errno is set.
5958  */
5959 static int
5960 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
5961                     void *arg)
5962 {
5963         enum rte_fdir_mode fdir_mode =
5964                 dev->data->dev_conf.fdir_conf.mode;
5965
5966         if (filter_op == RTE_ETH_FILTER_NOP)
5967                 return 0;
5968         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
5969             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
5970                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
5971                         dev->data->port_id, fdir_mode);
5972                 rte_errno = EINVAL;
5973                 return -rte_errno;
5974         }
5975         switch (filter_op) {
5976         case RTE_ETH_FILTER_ADD:
5977                 return flow_fdir_filter_add(dev, arg);
5978         case RTE_ETH_FILTER_UPDATE:
5979                 return flow_fdir_filter_update(dev, arg);
5980         case RTE_ETH_FILTER_DELETE:
5981                 return flow_fdir_filter_delete(dev, arg);
5982         case RTE_ETH_FILTER_FLUSH:
5983                 flow_fdir_filter_flush(dev);
5984                 break;
5985         case RTE_ETH_FILTER_INFO:
5986                 flow_fdir_info_get(dev, arg);
5987                 break;
5988         default:
5989                 DRV_LOG(DEBUG, "port %u unknown operation %u",
5990                         dev->data->port_id, filter_op);
5991                 rte_errno = EINVAL;
5992                 return -rte_errno;
5993         }
5994         return 0;
5995 }
5996
5997 /**
5998  * Manage filter operations.
5999  *
6000  * @param dev
6001  *   Pointer to Ethernet device structure.
6002  * @param filter_type
6003  *   Filter type.
6004  * @param filter_op
6005  *   Operation to perform.
6006  * @param arg
6007  *   Pointer to operation-specific structure.
6008  *
6009  * @return
6010  *   0 on success, a negative errno value otherwise and rte_errno is set.
6011  */
6012 int
6013 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
6014                      enum rte_filter_type filter_type,
6015                      enum rte_filter_op filter_op,
6016                      void *arg)
6017 {
6018         switch (filter_type) {
6019         case RTE_ETH_FILTER_GENERIC:
6020                 if (filter_op != RTE_ETH_FILTER_GET) {
6021                         rte_errno = EINVAL;
6022                         return -rte_errno;
6023                 }
6024                 *(const void **)arg = &mlx5_flow_ops;
6025                 return 0;
6026         case RTE_ETH_FILTER_FDIR:
6027                 return flow_fdir_ctrl_func(dev, filter_op, arg);
6028         default:
6029                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
6030                         dev->data->port_id, filter_type);
6031                 rte_errno = ENOTSUP;
6032                 return -rte_errno;
6033         }
6034         return 0;
6035 }
6036
6037 /**
6038  * Create the needed meter and suffix tables.
6039  *
6040  * @param[in] dev
6041  *   Pointer to Ethernet device.
6042  * @param[in] fm
6043  *   Pointer to the flow meter.
6044  *
6045  * @return
6046  *   Pointer to table set on success, NULL otherwise.
6047  */
6048 struct mlx5_meter_domains_infos *
6049 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
6050                           const struct mlx5_flow_meter *fm)
6051 {
6052         const struct mlx5_flow_driver_ops *fops;
6053
6054         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6055         return fops->create_mtr_tbls(dev, fm);
6056 }
6057
6058 /**
6059  * Destroy the meter table set.
6060  *
6061  * @param[in] dev
6062  *   Pointer to Ethernet device.
6063  * @param[in] tbl
6064  *   Pointer to the meter table set.
6065  *
6066  * @return
6067  *   0 on success.
6068  */
6069 int
6070 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
6071                            struct mlx5_meter_domains_infos *tbls)
6072 {
6073         const struct mlx5_flow_driver_ops *fops;
6074
6075         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6076         return fops->destroy_mtr_tbls(dev, tbls);
6077 }
6078
6079 /**
6080  * Create policer rules.
6081  *
6082  * @param[in] dev
6083  *   Pointer to Ethernet device.
6084  * @param[in] fm
6085  *   Pointer to flow meter structure.
6086  * @param[in] attr
6087  *   Pointer to flow attributes.
6088  *
6089  * @return
6090  *   0 on success, -1 otherwise.
6091  */
6092 int
6093 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
6094                                struct mlx5_flow_meter *fm,
6095                                const struct rte_flow_attr *attr)
6096 {
6097         const struct mlx5_flow_driver_ops *fops;
6098
6099         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6100         return fops->create_policer_rules(dev, fm, attr);
6101 }
6102
6103 /**
6104  * Destroy policer rules.
6105  *
6106  * @param[in] fm
6107  *   Pointer to flow meter structure.
6108  * @param[in] attr
6109  *   Pointer to flow attributes.
6110  *
6111  * @return
6112  *   0 on success, -1 otherwise.
6113  */
6114 int
6115 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
6116                                 struct mlx5_flow_meter *fm,
6117                                 const struct rte_flow_attr *attr)
6118 {
6119         const struct mlx5_flow_driver_ops *fops;
6120
6121         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6122         return fops->destroy_policer_rules(dev, fm, attr);
6123 }
6124
6125 /**
6126  * Allocate a counter.
6127  *
6128  * @param[in] dev
6129  *   Pointer to Ethernet device structure.
6130  *
6131  * @return
6132  *   Index to allocated counter  on success, 0 otherwise.
6133  */
6134 uint32_t
6135 mlx5_counter_alloc(struct rte_eth_dev *dev)
6136 {
6137         const struct mlx5_flow_driver_ops *fops;
6138         struct rte_flow_attr attr = { .transfer = 0 };
6139
6140         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6141                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6142                 return fops->counter_alloc(dev);
6143         }
6144         DRV_LOG(ERR,
6145                 "port %u counter allocate is not supported.",
6146                  dev->data->port_id);
6147         return 0;
6148 }
6149
6150 /**
6151  * Free a counter.
6152  *
6153  * @param[in] dev
6154  *   Pointer to Ethernet device structure.
6155  * @param[in] cnt
6156  *   Index to counter to be free.
6157  */
6158 void
6159 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
6160 {
6161         const struct mlx5_flow_driver_ops *fops;
6162         struct rte_flow_attr attr = { .transfer = 0 };
6163
6164         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6165                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6166                 fops->counter_free(dev, cnt);
6167                 return;
6168         }
6169         DRV_LOG(ERR,
6170                 "port %u counter free is not supported.",
6171                  dev->data->port_id);
6172 }
6173
6174 /**
6175  * Query counter statistics.
6176  *
6177  * @param[in] dev
6178  *   Pointer to Ethernet device structure.
6179  * @param[in] cnt
6180  *   Index to counter to query.
6181  * @param[in] clear
6182  *   Set to clear counter statistics.
6183  * @param[out] pkts
6184  *   The counter hits packets number to save.
6185  * @param[out] bytes
6186  *   The counter hits bytes number to save.
6187  *
6188  * @return
6189  *   0 on success, a negative errno value otherwise.
6190  */
6191 int
6192 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
6193                    bool clear, uint64_t *pkts, uint64_t *bytes)
6194 {
6195         const struct mlx5_flow_driver_ops *fops;
6196         struct rte_flow_attr attr = { .transfer = 0 };
6197
6198         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6199                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6200                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
6201         }
6202         DRV_LOG(ERR,
6203                 "port %u counter query is not supported.",
6204                  dev->data->port_id);
6205         return -ENOTSUP;
6206 }
6207
6208 #define MLX5_POOL_QUERY_FREQ_US 1000000
6209
6210 /**
6211  * Get number of all validate pools.
6212  *
6213  * @param[in] sh
6214  *   Pointer to mlx5_dev_ctx_shared object.
6215  *
6216  * @return
6217  *   The number of all validate pools.
6218  */
6219 static uint32_t
6220 mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh)
6221 {
6222         int i;
6223         uint32_t pools_n = 0;
6224
6225         for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i)
6226                 pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid);
6227         return pools_n;
6228 }
6229
6230 /**
6231  * Set the periodic procedure for triggering asynchronous batch queries for all
6232  * the counter pools.
6233  *
6234  * @param[in] sh
6235  *   Pointer to mlx5_dev_ctx_shared object.
6236  */
6237 void
6238 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
6239 {
6240         uint32_t pools_n, us;
6241
6242         pools_n = mlx5_get_all_valid_pool_count(sh);
6243         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
6244         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
6245         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
6246                 sh->cmng.query_thread_on = 0;
6247                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
6248         } else {
6249                 sh->cmng.query_thread_on = 1;
6250         }
6251 }
6252
6253 /**
6254  * The periodic procedure for triggering asynchronous batch queries for all the
6255  * counter pools. This function is probably called by the host thread.
6256  *
6257  * @param[in] arg
6258  *   The parameter for the alarm process.
6259  */
6260 void
6261 mlx5_flow_query_alarm(void *arg)
6262 {
6263         struct mlx5_dev_ctx_shared *sh = arg;
6264         struct mlx5_devx_obj *dcs;
6265         uint16_t offset;
6266         int ret;
6267         uint8_t batch = sh->cmng.batch;
6268         uint8_t age = sh->cmng.age;
6269         uint16_t pool_index = sh->cmng.pool_index;
6270         struct mlx5_pools_container *cont;
6271         struct mlx5_flow_counter_pool *pool;
6272         int cont_loop = MLX5_CCONT_TYPE_MAX;
6273
6274         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
6275                 goto set_alarm;
6276 next_container:
6277         cont = MLX5_CNT_CONTAINER(sh, batch, age);
6278         rte_spinlock_lock(&cont->resize_sl);
6279         if (!cont->pools) {
6280                 rte_spinlock_unlock(&cont->resize_sl);
6281                 /* Check if all the containers are empty. */
6282                 if (unlikely(--cont_loop == 0))
6283                         goto set_alarm;
6284                 batch ^= 0x1;
6285                 pool_index = 0;
6286                 if (batch == 0 && pool_index == 0) {
6287                         age ^= 0x1;
6288                         sh->cmng.batch = batch;
6289                         sh->cmng.age = age;
6290                 }
6291                 goto next_container;
6292         }
6293         pool = cont->pools[pool_index];
6294         rte_spinlock_unlock(&cont->resize_sl);
6295         if (pool->raw_hw)
6296                 /* There is a pool query in progress. */
6297                 goto set_alarm;
6298         pool->raw_hw =
6299                 LIST_FIRST(&sh->cmng.free_stat_raws);
6300         if (!pool->raw_hw)
6301                 /* No free counter statistics raw memory. */
6302                 goto set_alarm;
6303         dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
6304                                                               (&pool->a64_dcs);
6305         if (dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) {
6306                 /* Pool without valid counter. */
6307                 pool->raw_hw = NULL;
6308                 goto next_pool;
6309         }
6310         offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
6311         /*
6312          * Identify the counters released between query trigger and query
6313          * handle more effiecntly. The counter released in this gap period
6314          * should wait for a new round of query as the new arrived packets
6315          * will not be taken into account.
6316          */
6317         pool->query_gen++;
6318         ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
6319                                                offset, NULL, NULL,
6320                                                pool->raw_hw->mem_mng->dm->id,
6321                                                (void *)(uintptr_t)
6322                                                (pool->raw_hw->data + offset),
6323                                                sh->devx_comp,
6324                                                (uint64_t)(uintptr_t)pool);
6325         if (ret) {
6326                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
6327                         " %d", pool->min_dcs->id);
6328                 pool->raw_hw = NULL;
6329                 goto set_alarm;
6330         }
6331         pool->raw_hw->min_dcs_id = dcs->id;
6332         LIST_REMOVE(pool->raw_hw, next);
6333         sh->cmng.pending_queries++;
6334 next_pool:
6335         pool_index++;
6336         if (pool_index >= rte_atomic16_read(&cont->n_valid)) {
6337                 batch ^= 0x1;
6338                 pool_index = 0;
6339                 if (batch == 0 && pool_index == 0)
6340                         age ^= 0x1;
6341         }
6342 set_alarm:
6343         sh->cmng.batch = batch;
6344         sh->cmng.pool_index = pool_index;
6345         sh->cmng.age = age;
6346         mlx5_set_query_alarm(sh);
6347 }
6348
6349 /**
6350  * Check and callback event for new aged flow in the counter pool
6351  *
6352  * @param[in] sh
6353  *   Pointer to mlx5_dev_ctx_shared object.
6354  * @param[in] pool
6355  *   Pointer to Current counter pool.
6356  */
6357 static void
6358 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6359                    struct mlx5_flow_counter_pool *pool)
6360 {
6361         struct mlx5_priv *priv;
6362         struct mlx5_flow_counter *cnt;
6363         struct mlx5_age_info *age_info;
6364         struct mlx5_age_param *age_param;
6365         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6366         struct mlx5_counter_stats_raw *prev = pool->raw;
6367         uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10);
6368         uint32_t i;
6369
6370         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6371                 cnt = MLX5_POOL_GET_CNT(pool, i);
6372                 age_param = MLX5_CNT_TO_AGE(cnt);
6373                 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE)
6374                         continue;
6375                 if (cur->data[i].hits != prev->data[i].hits) {
6376                         age_param->expire = curr + age_param->timeout;
6377                         continue;
6378                 }
6379                 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2))
6380                         continue;
6381                 /**
6382                  * Hold the lock first, or if between the
6383                  * state AGE_TMOUT and tailq operation the
6384                  * release happened, the release procedure
6385                  * may delete a non-existent tailq node.
6386                  */
6387                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6388                 age_info = GET_PORT_AGE_INFO(priv);
6389                 rte_spinlock_lock(&age_info->aged_sl);
6390                 /* If the cpmset fails, release happens. */
6391                 if (rte_atomic16_cmpset((volatile uint16_t *)
6392                                         &age_param->state,
6393                                         AGE_CANDIDATE,
6394                                         AGE_TMOUT) ==
6395                                         AGE_CANDIDATE) {
6396                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6397                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6398                 }
6399                 rte_spinlock_unlock(&age_info->aged_sl);
6400         }
6401         for (i = 0; i < sh->max_port; i++) {
6402                 age_info = &sh->port[i].age_info;
6403                 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW))
6404                         continue;
6405                 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER))
6406                         rte_eth_dev_callback_process
6407                                 (&rte_eth_devices[sh->port[i].devx_ih_port_id],
6408                                 RTE_ETH_EVENT_FLOW_AGED, NULL);
6409                 age_info->flags = 0;
6410         }
6411 }
6412
6413 /**
6414  * Handler for the HW respond about ready values from an asynchronous batch
6415  * query. This function is probably called by the host thread.
6416  *
6417  * @param[in] sh
6418  *   The pointer to the shared device context.
6419  * @param[in] async_id
6420  *   The Devx async ID.
6421  * @param[in] status
6422  *   The status of the completion.
6423  */
6424 void
6425 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6426                                   uint64_t async_id, int status)
6427 {
6428         struct mlx5_flow_counter_pool *pool =
6429                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6430         struct mlx5_counter_stats_raw *raw_to_free;
6431         uint8_t age = !!IS_AGE_POOL(pool);
6432         uint8_t query_gen = pool->query_gen ^ 1;
6433         struct mlx5_pools_container *cont =
6434                 MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool), age);
6435
6436         if (unlikely(status)) {
6437                 raw_to_free = pool->raw_hw;
6438         } else {
6439                 raw_to_free = pool->raw;
6440                 if (IS_AGE_POOL(pool))
6441                         mlx5_flow_aging_check(sh, pool);
6442                 rte_spinlock_lock(&pool->sl);
6443                 pool->raw = pool->raw_hw;
6444                 rte_spinlock_unlock(&pool->sl);
6445                 /* Be sure the new raw counters data is updated in memory. */
6446                 rte_io_wmb();
6447                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6448                         rte_spinlock_lock(&cont->csl);
6449                         TAILQ_CONCAT(&cont->counters,
6450                                      &pool->counters[query_gen], next);
6451                         rte_spinlock_unlock(&cont->csl);
6452                 }
6453         }
6454         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6455         pool->raw_hw = NULL;
6456         sh->cmng.pending_queries--;
6457 }
6458
6459 /**
6460  * Translate the rte_flow group index to HW table value.
6461  *
6462  * @param[in] attributes
6463  *   Pointer to flow attributes
6464  * @param[in] external
6465  *   Value is part of flow rule created by request external to PMD.
6466  * @param[in] group
6467  *   rte_flow group index value.
6468  * @param[out] fdb_def_rule
6469  *   Whether fdb jump to table 1 is configured.
6470  * @param[out] table
6471  *   HW table value.
6472  * @param[out] error
6473  *   Pointer to error structure.
6474  *
6475  * @return
6476  *   0 on success, a negative errno value otherwise and rte_errno is set.
6477  */
6478 int
6479 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external,
6480                          uint32_t group, bool fdb_def_rule, uint32_t *table,
6481                          struct rte_flow_error *error)
6482 {
6483         if (attributes->transfer && external && fdb_def_rule) {
6484                 if (group == UINT32_MAX)
6485                         return rte_flow_error_set
6486                                                 (error, EINVAL,
6487                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6488                                                  NULL,
6489                                                  "group index not supported");
6490                 *table = group + 1;
6491         } else {
6492                 *table = group;
6493         }
6494         return 0;
6495 }
6496
6497 /**
6498  * Discover availability of metadata reg_c's.
6499  *
6500  * Iteratively use test flows to check availability.
6501  *
6502  * @param[in] dev
6503  *   Pointer to the Ethernet device structure.
6504  *
6505  * @return
6506  *   0 on success, a negative errno value otherwise and rte_errno is set.
6507  */
6508 int
6509 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6510 {
6511         struct mlx5_priv *priv = dev->data->dev_private;
6512         struct mlx5_dev_config *config = &priv->config;
6513         enum modify_reg idx;
6514         int n = 0;
6515
6516         /* reg_c[0] and reg_c[1] are reserved. */
6517         config->flow_mreg_c[n++] = REG_C_0;
6518         config->flow_mreg_c[n++] = REG_C_1;
6519         /* Discover availability of other reg_c's. */
6520         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6521                 struct rte_flow_attr attr = {
6522                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6523                         .priority = MLX5_FLOW_PRIO_RSVD,
6524                         .ingress = 1,
6525                 };
6526                 struct rte_flow_item items[] = {
6527                         [0] = {
6528                                 .type = RTE_FLOW_ITEM_TYPE_END,
6529                         },
6530                 };
6531                 struct rte_flow_action actions[] = {
6532                         [0] = {
6533                                 .type = (enum rte_flow_action_type)
6534                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6535                                 .conf = &(struct mlx5_flow_action_copy_mreg){
6536                                         .src = REG_C_1,
6537                                         .dst = idx,
6538                                 },
6539                         },
6540                         [1] = {
6541                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6542                                 .conf = &(struct rte_flow_action_jump){
6543                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6544                                 },
6545                         },
6546                         [2] = {
6547                                 .type = RTE_FLOW_ACTION_TYPE_END,
6548                         },
6549                 };
6550                 uint32_t flow_idx;
6551                 struct rte_flow *flow;
6552                 struct rte_flow_error error;
6553
6554                 if (!config->dv_flow_en)
6555                         break;
6556                 /* Create internal flow, validation skips copy action. */
6557                 flow_idx = flow_list_create(dev, NULL, &attr, items,
6558                                             actions, false, &error);
6559                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6560                                       flow_idx);
6561                 if (!flow)
6562                         continue;
6563                 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
6564                         config->flow_mreg_c[n++] = idx;
6565                 flow_list_destroy(dev, NULL, flow_idx);
6566         }
6567         for (; n < MLX5_MREG_C_NUM; ++n)
6568                 config->flow_mreg_c[n] = REG_NON;
6569         return 0;
6570 }
6571
6572 /**
6573  * Dump flow raw hw data to file
6574  *
6575  * @param[in] dev
6576  *    The pointer to Ethernet device.
6577  * @param[in] file
6578  *   A pointer to a file for output.
6579  * @param[out] error
6580  *   Perform verbose error reporting if not NULL. PMDs initialize this
6581  *   structure in case of error only.
6582  * @return
6583  *   0 on success, a nagative value otherwise.
6584  */
6585 int
6586 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6587                    FILE *file,
6588                    struct rte_flow_error *error __rte_unused)
6589 {
6590         struct mlx5_priv *priv = dev->data->dev_private;
6591         struct mlx5_dev_ctx_shared *sh = priv->sh;
6592
6593         return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6594                                        sh->tx_domain, file);
6595 }
6596
6597 /**
6598  * Get aged-out flows.
6599  *
6600  * @param[in] dev
6601  *   Pointer to the Ethernet device structure.
6602  * @param[in] context
6603  *   The address of an array of pointers to the aged-out flows contexts.
6604  * @param[in] nb_countexts
6605  *   The length of context array pointers.
6606  * @param[out] error
6607  *   Perform verbose error reporting if not NULL. Initialized in case of
6608  *   error only.
6609  *
6610  * @return
6611  *   how many contexts get in success, otherwise negative errno value.
6612  *   if nb_contexts is 0, return the amount of all aged contexts.
6613  *   if nb_contexts is not 0 , return the amount of aged flows reported
6614  *   in the context array.
6615  */
6616 int
6617 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6618                         uint32_t nb_contexts, struct rte_flow_error *error)
6619 {
6620         const struct mlx5_flow_driver_ops *fops;
6621         struct rte_flow_attr attr = { .transfer = 0 };
6622
6623         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6624                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6625                 return fops->get_aged_flows(dev, contexts, nb_contexts,
6626                                                     error);
6627         }
6628         DRV_LOG(ERR,
6629                 "port %u get aged flows is not supported.",
6630                  dev->data->port_id);
6631         return -ENOTSUP;
6632 }