drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint8_t optional;
 104         /**< optional expand field. Default 0 to expand, 1 not go deeper. */
 105 };
 106
 107 /** Object returned by mlx5_flow_expand_rss(). */
 108 struct mlx5_flow_expand_rss {
 109         uint32_t entries;
 110         /**< Number of entries @p patterns and @p priorities. */
 111         struct {
 112                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 113                 uint32_t priority; /**< Priority offset for each expansion. */
 114         } entry[];
 115 };
 116
 117 static void
 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 119
 120 static bool
 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 122 {
 123         switch (item->type) {
 124         case RTE_FLOW_ITEM_TYPE_ETH:
 125         case RTE_FLOW_ITEM_TYPE_VLAN:
 126         case RTE_FLOW_ITEM_TYPE_IPV4:
 127         case RTE_FLOW_ITEM_TYPE_IPV6:
 128         case RTE_FLOW_ITEM_TYPE_UDP:
 129         case RTE_FLOW_ITEM_TYPE_TCP:
 130         case RTE_FLOW_ITEM_TYPE_VXLAN:
 131         case RTE_FLOW_ITEM_TYPE_NVGRE:
 132         case RTE_FLOW_ITEM_TYPE_GRE:
 133         case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 return true;
 135         default:
 136                 break;
 137         }
 138         return false;
 139 }
 140
 141 static enum rte_flow_item_type
 142 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 143 {
 144         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 145         uint16_t ether_type = 0;
 146         uint16_t ether_type_m;
 147         uint8_t ip_next_proto = 0;
 148         uint8_t ip_next_proto_m;
 149
 150         if (item == NULL || item->spec == NULL)
 151                 return ret;
 152         switch (item->type) {
 153         case RTE_FLOW_ITEM_TYPE_ETH:
 154                 if (item->mask)
 155                         ether_type_m = ((const struct rte_flow_item_eth *)
 156                                                 (item->mask))->type;
 157                 else
 158                         ether_type_m = rte_flow_item_eth_mask.type;
 159                 if (ether_type_m != RTE_BE16(0xFFFF))
 160                         break;
 161                 ether_type = ((const struct rte_flow_item_eth *)
 162                                 (item->spec))->type;
 163                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 164                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 165                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 166                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 167                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 168                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 169                 else
 170                         ret = RTE_FLOW_ITEM_TYPE_END;
 171                 break;
 172         case RTE_FLOW_ITEM_TYPE_VLAN:
 173                 if (item->mask)
 174                         ether_type_m = ((const struct rte_flow_item_vlan *)
 175                                                 (item->mask))->inner_type;
 176                 else
 177                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 178                 if (ether_type_m != RTE_BE16(0xFFFF))
 179                         break;
 180                 ether_type = ((const struct rte_flow_item_vlan *)
 181                                 (item->spec))->inner_type;
 182                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 183                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 185                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 186                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 187                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 188                 else
 189                         ret = RTE_FLOW_ITEM_TYPE_END;
 190                 break;
 191         case RTE_FLOW_ITEM_TYPE_IPV4:
 192                 if (item->mask)
 193                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 194                                         (item->mask))->hdr.next_proto_id;
 195                 else
 196                         ip_next_proto_m =
 197                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 198                 if (ip_next_proto_m != 0xFF)
 199                         break;
 200                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 201                                 (item->spec))->hdr.next_proto_id;
 202                 if (ip_next_proto == IPPROTO_UDP)
 203                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 204                 else if (ip_next_proto == IPPROTO_TCP)
 205                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 206                 else if (ip_next_proto == IPPROTO_IP)
 207                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 208                 else if (ip_next_proto == IPPROTO_IPV6)
 209                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 210                 else
 211                         ret = RTE_FLOW_ITEM_TYPE_END;
 212                 break;
 213         case RTE_FLOW_ITEM_TYPE_IPV6:
 214                 if (item->mask)
 215                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 216                                                 (item->mask))->hdr.proto;
 217                 else
 218                         ip_next_proto_m =
 219                                 rte_flow_item_ipv6_mask.hdr.proto;
 220                 if (ip_next_proto_m != 0xFF)
 221                         break;
 222                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 223                                 (item->spec))->hdr.proto;
 224                 if (ip_next_proto == IPPROTO_UDP)
 225                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 226                 else if (ip_next_proto == IPPROTO_TCP)
 227                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 228                 else if (ip_next_proto == IPPROTO_IP)
 229                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 230                 else if (ip_next_proto == IPPROTO_IPV6)
 231                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 232                 else
 233                         ret = RTE_FLOW_ITEM_TYPE_END;
 234                 break;
 235         default:
 236                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 237                 break;
 238         }
 239         return ret;
 240 }
 241
 242 #define MLX5_RSS_EXP_ELT_N 16
 243
 244 /**
 245  * Expand RSS flows into several possible flows according to the RSS hash
 246  * fields requested and the driver capabilities.
 247  *
 248  * @param[out] buf
 249  *   Buffer to store the result expansion.
 250  * @param[in] size
 251  *   Buffer size in bytes. If 0, @p buf can be NULL.
 252  * @param[in] pattern
 253  *   User flow pattern.
 254  * @param[in] types
 255  *   RSS types to expand (see ETH_RSS_* definitions).
 256  * @param[in] graph
 257  *   Input graph to expand @p pattern according to @p types.
 258  * @param[in] graph_root_index
 259  *   Index of root node in @p graph, typically 0.
 260  *
 261  * @return
 262  *   A positive value representing the size of @p buf in bytes regardless of
 263  *   @p size on success, a negative errno value otherwise and rte_errno is
 264  *   set, the following errors are defined:
 265  *
 266  *   -E2BIG: graph-depth @p graph is too deep.
 267  */
 268 static int
 269 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 270                      const struct rte_flow_item *pattern, uint64_t types,
 271                      const struct mlx5_flow_expand_node graph[],
 272                      int graph_root_index)
 273 {
 274         const struct rte_flow_item *item;
 275         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 276         const int *next_node;
 277         const int *stack[MLX5_RSS_EXP_ELT_N];
 278         int stack_pos = 0;
 279         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 280         unsigned int i;
 281         size_t lsize;
 282         size_t user_pattern_size = 0;
 283         void *addr = NULL;
 284         const struct mlx5_flow_expand_node *next = NULL;
 285         struct rte_flow_item missed_item;
 286         int missed = 0;
 287         int elt = 0;
 288         const struct rte_flow_item *last_item = NULL;
 289
 290         memset(&missed_item, 0, sizeof(missed_item));
 291         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 292                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 293         if (lsize <= size) {
 294                 buf->entry[0].priority = 0;
 295                 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 296                 buf->entries = 0;
 297                 addr = buf->entry[0].pattern;
 298         }
 299         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 300                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 301                         user_pattern_size += sizeof(*item);
 302                         continue;
 303                 }
 304                 last_item = item;
 305                 for (i = 0; node->next && node->next[i]; ++i) {
 306                         next = &graph[node->next[i]];
 307                         if (next->type == item->type)
 308                                 break;
 309                 }
 310                 if (next)
 311                         node = next;
 312                 user_pattern_size += sizeof(*item);
 313         }
 314         user_pattern_size += sizeof(*item); /* Handle END item. */
 315         lsize += user_pattern_size;
 316         /* Copy the user pattern in the first entry of the buffer. */
 317         if (lsize <= size) {
 318                 rte_memcpy(addr, pattern, user_pattern_size);
 319                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 320                 buf->entries = 1;
 321         }
 322         /* Start expanding. */
 323         memset(flow_items, 0, sizeof(flow_items));
 324         user_pattern_size -= sizeof(*item);
 325         /*
 326          * Check if the last valid item has spec set, need complete pattern,
 327          * and the pattern can be used for expansion.
 328          */
 329         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 330         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 331                 /* Item type END indicates expansion is not required. */
 332                 return lsize;
 333         }
 334         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 335                 next = NULL;
 336                 missed = 1;
 337                 for (i = 0; node->next && node->next[i]; ++i) {
 338                         next = &graph[node->next[i]];
 339                         if (next->type == missed_item.type) {
 340                                 flow_items[0].type = missed_item.type;
 341                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 342                                 break;
 343                         }
 344                         next = NULL;
 345                 }
 346         }
 347         if (next && missed) {
 348                 elt = 2; /* missed item + item end. */
 349                 node = next;
 350                 lsize += elt * sizeof(*item) + user_pattern_size;
 351                 if ((node->rss_types & types) && lsize <= size) {
 352                         buf->entry[buf->entries].priority = 1;
 353                         buf->entry[buf->entries].pattern = addr;
 354                         buf->entries++;
 355                         rte_memcpy(addr, buf->entry[0].pattern,
 356                                    user_pattern_size);
 357                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 358                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 359                         addr = (void *)(((uintptr_t)addr) +
 360                                         elt * sizeof(*item));
 361                 }
 362         }
 363         memset(flow_items, 0, sizeof(flow_items));
 364         next_node = node->next;
 365         stack[stack_pos] = next_node;
 366         node = next_node ? &graph[*next_node] : NULL;
 367         while (node) {
 368                 flow_items[stack_pos].type = node->type;
 369                 if (node->rss_types & types) {
 370                         /*
 371                          * compute the number of items to copy from the
 372                          * expansion and copy it.
 373                          * When the stack_pos is 0, there are 1 element in it,
 374                          * plus the addition END item.
 375                          */
 376                         elt = stack_pos + 2;
 377                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 378                         lsize += elt * sizeof(*item) + user_pattern_size;
 379                         if (lsize <= size) {
 380                                 size_t n = elt * sizeof(*item);
 381
 382                                 buf->entry[buf->entries].priority =
 383                                         stack_pos + 1 + missed;
 384                                 buf->entry[buf->entries].pattern = addr;
 385                                 buf->entries++;
 386                                 rte_memcpy(addr, buf->entry[0].pattern,
 387                                            user_pattern_size);
 388                                 addr = (void *)(((uintptr_t)addr) +
 389                                                 user_pattern_size);
 390                                 rte_memcpy(addr, &missed_item,
 391                                            missed * sizeof(*item));
 392                                 addr = (void *)(((uintptr_t)addr) +
 393                                         missed * sizeof(*item));
 394                                 rte_memcpy(addr, flow_items, n);
 395                                 addr = (void *)(((uintptr_t)addr) + n);
 396                         }
 397                 }
 398                 /* Go deeper. */
 399                 if (!node->optional && node->next) {
 400                         next_node = node->next;
 401                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 402                                 rte_errno = E2BIG;
 403                                 return -rte_errno;
 404                         }
 405                         stack[stack_pos] = next_node;
 406                 } else if (*(next_node + 1)) {
 407                         /* Follow up with the next possibility. */
 408                         ++next_node;
 409                 } else {
 410                         /* Move to the next path. */
 411                         if (stack_pos)
 412                                 next_node = stack[--stack_pos];
 413                         next_node++;
 414                         stack[stack_pos] = next_node;
 415                 }
 416                 node = *next_node ? &graph[*next_node] : NULL;
 417         };
 418         return lsize;
 419 }
 420
 421 enum mlx5_expansion {
 422         MLX5_EXPANSION_ROOT,
 423         MLX5_EXPANSION_ROOT_OUTER,
 424         MLX5_EXPANSION_ROOT_ETH_VLAN,
 425         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 426         MLX5_EXPANSION_OUTER_ETH,
 427         MLX5_EXPANSION_OUTER_ETH_VLAN,
 428         MLX5_EXPANSION_OUTER_VLAN,
 429         MLX5_EXPANSION_OUTER_IPV4,
 430         MLX5_EXPANSION_OUTER_IPV4_UDP,
 431         MLX5_EXPANSION_OUTER_IPV4_TCP,
 432         MLX5_EXPANSION_OUTER_IPV6,
 433         MLX5_EXPANSION_OUTER_IPV6_UDP,
 434         MLX5_EXPANSION_OUTER_IPV6_TCP,
 435         MLX5_EXPANSION_VXLAN,
 436         MLX5_EXPANSION_VXLAN_GPE,
 437         MLX5_EXPANSION_GRE,
 438         MLX5_EXPANSION_NVGRE,
 439         MLX5_EXPANSION_GRE_KEY,
 440         MLX5_EXPANSION_MPLS,
 441         MLX5_EXPANSION_ETH,
 442         MLX5_EXPANSION_ETH_VLAN,
 443         MLX5_EXPANSION_VLAN,
 444         MLX5_EXPANSION_IPV4,
 445         MLX5_EXPANSION_IPV4_UDP,
 446         MLX5_EXPANSION_IPV4_TCP,
 447         MLX5_EXPANSION_IPV6,
 448         MLX5_EXPANSION_IPV6_UDP,
 449         MLX5_EXPANSION_IPV6_TCP,
 450 };
 451
 452 /** Supported expansion of items. */
 453 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 454         [MLX5_EXPANSION_ROOT] = {
 455                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 456                                                   MLX5_EXPANSION_IPV4,
 457                                                   MLX5_EXPANSION_IPV6),
 458                 .type = RTE_FLOW_ITEM_TYPE_END,
 459         },
 460         [MLX5_EXPANSION_ROOT_OUTER] = {
 461                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 462                                                   MLX5_EXPANSION_OUTER_IPV4,
 463                                                   MLX5_EXPANSION_OUTER_IPV6),
 464                 .type = RTE_FLOW_ITEM_TYPE_END,
 465         },
 466         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 467                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 468                 .type = RTE_FLOW_ITEM_TYPE_END,
 469         },
 470         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 471                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 472                                                 (MLX5_EXPANSION_OUTER_ETH_VLAN),
 473                 .type = RTE_FLOW_ITEM_TYPE_END,
 474         },
 475         [MLX5_EXPANSION_OUTER_ETH] = {
 476                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 477                                                   MLX5_EXPANSION_OUTER_IPV6,
 478                                                   MLX5_EXPANSION_MPLS),
 479                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 480                 .rss_types = 0,
 481         },
 482         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 483                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 484                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 485                 .rss_types = 0,
 486         },
 487         [MLX5_EXPANSION_OUTER_VLAN] = {
 488                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 489                                                   MLX5_EXPANSION_OUTER_IPV6),
 490                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 491         },
 492         [MLX5_EXPANSION_OUTER_IPV4] = {
 493                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 494                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 495                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 496                          MLX5_EXPANSION_GRE,
 497                          MLX5_EXPANSION_NVGRE,
 498                          MLX5_EXPANSION_IPV4,
 499                          MLX5_EXPANSION_IPV6),
 500                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 501                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 502                         ETH_RSS_NONFRAG_IPV4_OTHER,
 503         },
 504         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 505                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 506                                                   MLX5_EXPANSION_VXLAN_GPE),
 507                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 508                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 509         },
 510         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 511                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 512                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 513         },
 514         [MLX5_EXPANSION_OUTER_IPV6] = {
 515                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 516                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 517                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 518                          MLX5_EXPANSION_IPV4,
 519                          MLX5_EXPANSION_IPV6,
 520                          MLX5_EXPANSION_GRE,
 521                          MLX5_EXPANSION_NVGRE),
 522                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 523                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 524                         ETH_RSS_NONFRAG_IPV6_OTHER,
 525         },
 526         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 527                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 528                                                   MLX5_EXPANSION_VXLAN_GPE),
 529                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 530                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 531         },
 532         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 533                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 534                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 535         },
 536         [MLX5_EXPANSION_VXLAN] = {
 537                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 538                                                   MLX5_EXPANSION_IPV4,
 539                                                   MLX5_EXPANSION_IPV6),
 540                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 541         },
 542         [MLX5_EXPANSION_VXLAN_GPE] = {
 543                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 544                                                   MLX5_EXPANSION_IPV4,
 545                                                   MLX5_EXPANSION_IPV6),
 546                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 547         },
 548         [MLX5_EXPANSION_GRE] = {
 549                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 550                                                   MLX5_EXPANSION_IPV6,
 551                                                   MLX5_EXPANSION_GRE_KEY),
 552                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 553         },
 554         [MLX5_EXPANSION_GRE_KEY] = {
 555                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 556                                                   MLX5_EXPANSION_IPV6),
 557                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 558                 .optional = 1,
 559         },
 560         [MLX5_EXPANSION_NVGRE] = {
 561                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 562                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 563         },
 564         [MLX5_EXPANSION_MPLS] = {
 565                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 566                                                   MLX5_EXPANSION_IPV6),
 567                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 568         },
 569         [MLX5_EXPANSION_ETH] = {
 570                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 571                                                   MLX5_EXPANSION_IPV6),
 572                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 573         },
 574         [MLX5_EXPANSION_ETH_VLAN] = {
 575                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 576                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 577         },
 578         [MLX5_EXPANSION_VLAN] = {
 579                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 580                                                   MLX5_EXPANSION_IPV6),
 581                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 582         },
 583         [MLX5_EXPANSION_IPV4] = {
 584                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 585                                                   MLX5_EXPANSION_IPV4_TCP),
 586                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 587                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 588                         ETH_RSS_NONFRAG_IPV4_OTHER,
 589         },
 590         [MLX5_EXPANSION_IPV4_UDP] = {
 591                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 592                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 593         },
 594         [MLX5_EXPANSION_IPV4_TCP] = {
 595                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 596                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 597         },
 598         [MLX5_EXPANSION_IPV6] = {
 599                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 600                                                   MLX5_EXPANSION_IPV6_TCP),
 601                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 602                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 603                         ETH_RSS_NONFRAG_IPV6_OTHER,
 604         },
 605         [MLX5_EXPANSION_IPV6_UDP] = {
 606                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 607                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 608         },
 609         [MLX5_EXPANSION_IPV6_TCP] = {
 610                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 611                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 612         },
 613 };
 614
 615 static struct rte_flow_action_handle *
 616 mlx5_action_handle_create(struct rte_eth_dev *dev,
 617                           const struct rte_flow_indir_action_conf *conf,
 618                           const struct rte_flow_action *action,
 619                           struct rte_flow_error *error);
 620 static int mlx5_action_handle_destroy
 621                                 (struct rte_eth_dev *dev,
 622                                  struct rte_flow_action_handle *handle,
 623                                  struct rte_flow_error *error);
 624 static int mlx5_action_handle_update
 625                                 (struct rte_eth_dev *dev,
 626                                  struct rte_flow_action_handle *handle,
 627                                  const void *update,
 628                                  struct rte_flow_error *error);
 629 static int mlx5_action_handle_query
 630                                 (struct rte_eth_dev *dev,
 631                                  const struct rte_flow_action_handle *handle,
 632                                  void *data,
 633                                  struct rte_flow_error *error);
 634 static int
 635 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 636                     struct rte_flow_tunnel *app_tunnel,
 637                     struct rte_flow_action **actions,
 638                     uint32_t *num_of_actions,
 639                     struct rte_flow_error *error);
 640 static int
 641 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 642                        struct rte_flow_tunnel *app_tunnel,
 643                        struct rte_flow_item **items,
 644                        uint32_t *num_of_items,
 645                        struct rte_flow_error *error);
 646 static int
 647 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 648                               struct rte_flow_item *pmd_items,
 649                               uint32_t num_items, struct rte_flow_error *err);
 650 static int
 651 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 652                                 struct rte_flow_action *pmd_actions,
 653                                 uint32_t num_actions,
 654                                 struct rte_flow_error *err);
 655 static int
 656 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 657                                   struct rte_mbuf *m,
 658                                   struct rte_flow_restore_info *info,
 659                                   struct rte_flow_error *err);
 660
 661 static const struct rte_flow_ops mlx5_flow_ops = {
 662         .validate = mlx5_flow_validate,
 663         .create = mlx5_flow_create,
 664         .destroy = mlx5_flow_destroy,
 665         .flush = mlx5_flow_flush,
 666         .isolate = mlx5_flow_isolate,
 667         .query = mlx5_flow_query,
 668         .dev_dump = mlx5_flow_dev_dump,
 669         .get_aged_flows = mlx5_flow_get_aged_flows,
 670         .action_handle_create = mlx5_action_handle_create,
 671         .action_handle_destroy = mlx5_action_handle_destroy,
 672         .action_handle_update = mlx5_action_handle_update,
 673         .action_handle_query = mlx5_action_handle_query,
 674         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 675         .tunnel_match = mlx5_flow_tunnel_match,
 676         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 677         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 678         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 679 };
 680
 681 /* Tunnel information. */
 682 struct mlx5_flow_tunnel_info {
 683         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 684         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 685 };
 686
 687 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 688         {
 689                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 690                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 691         },
 692         {
 693                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 694                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 695         },
 696         {
 697                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 698                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 699         },
 700         {
 701                 .tunnel = MLX5_FLOW_LAYER_GRE,
 702                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 703         },
 704         {
 705                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 706                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 707         },
 708         {
 709                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 710                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 711         },
 712         {
 713                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 714                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 715         },
 716         {
 717                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 718                 .ptype = RTE_PTYPE_TUNNEL_IP,
 719         },
 720         {
 721                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 722                 .ptype = RTE_PTYPE_TUNNEL_IP,
 723         },
 724         {
 725                 .tunnel = MLX5_FLOW_LAYER_GTP,
 726                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 727         },
 728 };
 729
 730
 731
 732 /**
 733  * Translate tag ID to register.
 734  *
 735  * @param[in] dev
 736  *   Pointer to the Ethernet device structure.
 737  * @param[in] feature
 738  *   The feature that request the register.
 739  * @param[in] id
 740  *   The request register ID.
 741  * @param[out] error
 742  *   Error description in case of any.
 743  *
 744  * @return
 745  *   The request register on success, a negative errno
 746  *   value otherwise and rte_errno is set.
 747  */
 748 int
 749 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 750                      enum mlx5_feature_name feature,
 751                      uint32_t id,
 752                      struct rte_flow_error *error)
 753 {
 754         struct mlx5_priv *priv = dev->data->dev_private;
 755         struct mlx5_dev_config *config = &priv->config;
 756         enum modify_reg start_reg;
 757         bool skip_mtr_reg = false;
 758
 759         switch (feature) {
 760         case MLX5_HAIRPIN_RX:
 761                 return REG_B;
 762         case MLX5_HAIRPIN_TX:
 763                 return REG_A;
 764         case MLX5_METADATA_RX:
 765                 switch (config->dv_xmeta_en) {
 766                 case MLX5_XMETA_MODE_LEGACY:
 767                         return REG_B;
 768                 case MLX5_XMETA_MODE_META16:
 769                         return REG_C_0;
 770                 case MLX5_XMETA_MODE_META32:
 771                         return REG_C_1;
 772                 }
 773                 break;
 774         case MLX5_METADATA_TX:
 775                 return REG_A;
 776         case MLX5_METADATA_FDB:
 777                 switch (config->dv_xmeta_en) {
 778                 case MLX5_XMETA_MODE_LEGACY:
 779                         return REG_NON;
 780                 case MLX5_XMETA_MODE_META16:
 781                         return REG_C_0;
 782                 case MLX5_XMETA_MODE_META32:
 783                         return REG_C_1;
 784                 }
 785                 break;
 786         case MLX5_FLOW_MARK:
 787                 switch (config->dv_xmeta_en) {
 788                 case MLX5_XMETA_MODE_LEGACY:
 789                         return REG_NON;
 790                 case MLX5_XMETA_MODE_META16:
 791                         return REG_C_1;
 792                 case MLX5_XMETA_MODE_META32:
 793                         return REG_C_0;
 794                 }
 795                 break;
 796         case MLX5_MTR_ID:
 797                 /*
 798                  * If meter color and meter id share one register, flow match
 799                  * should use the meter color register for match.
 800                  */
 801                 if (priv->mtr_reg_share)
 802                         return priv->mtr_color_reg;
 803                 else
 804                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 805                                REG_C_3;
 806         case MLX5_MTR_COLOR:
 807         case MLX5_ASO_FLOW_HIT:
 808         case MLX5_ASO_CONNTRACK:
 809                 /* All features use the same REG_C. */
 810                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 811                 return priv->mtr_color_reg;
 812         case MLX5_COPY_MARK:
 813                 /*
 814                  * Metadata COPY_MARK register using is in meter suffix sub
 815                  * flow while with meter. It's safe to share the same register.
 816                  */
 817                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 818         case MLX5_APP_TAG:
 819                 /*
 820                  * If meter is enable, it will engage the register for color
 821                  * match and flow match. If meter color match is not using the
 822                  * REG_C_2, need to skip the REG_C_x be used by meter color
 823                  * match.
 824                  * If meter is disable, free to use all available registers.
 825                  */
 826                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 827                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 828                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 829                 if (id > (uint32_t)(REG_C_7 - start_reg))
 830                         return rte_flow_error_set(error, EINVAL,
 831                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 832                                                   NULL, "invalid tag id");
 833                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 834                         return rte_flow_error_set(error, ENOTSUP,
 835                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 836                                                   NULL, "unsupported tag id");
 837                 /*
 838                  * This case means meter is using the REG_C_x great than 2.
 839                  * Take care not to conflict with meter color REG_C_x.
 840                  * If the available index REG_C_y >= REG_C_x, skip the
 841                  * color register.
 842                  */
 843                 if (skip_mtr_reg && config->flow_mreg_c
 844                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 845                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 846                                 return rte_flow_error_set(error, EINVAL,
 847                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 848                                                         NULL, "invalid tag id");
 849                         if (config->flow_mreg_c
 850                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 851                                 return config->flow_mreg_c
 852                                                [id + 1 + start_reg - REG_C_0];
 853                         return rte_flow_error_set(error, ENOTSUP,
 854                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 855                                                   NULL, "unsupported tag id");
 856                 }
 857                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 858         }
 859         MLX5_ASSERT(false);
 860         return rte_flow_error_set(error, EINVAL,
 861                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 862                                   NULL, "invalid feature name");
 863 }
 864
 865 /**
 866  * Check extensive flow metadata register support.
 867  *
 868  * @param dev
 869  *   Pointer to rte_eth_dev structure.
 870  *
 871  * @return
 872  *   True if device supports extensive flow metadata register, otherwise false.
 873  */
 874 bool
 875 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 876 {
 877         struct mlx5_priv *priv = dev->data->dev_private;
 878         struct mlx5_dev_config *config = &priv->config;
 879
 880         /*
 881          * Having available reg_c can be regarded inclusively as supporting
 882          * extensive flow metadata register, which could mean,
 883          * - metadata register copy action by modify header.
 884          * - 16 modify header actions is supported.
 885          * - reg_c's are preserved across different domain (FDB and NIC) on
 886          *   packet loopback by flow lookup miss.
 887          */
 888         return config->flow_mreg_c[2] != REG_NON;
 889 }
 890
 891 /**
 892  * Get the lowest priority.
 893  *
 894  * @param[in] dev
 895  *   Pointer to the Ethernet device structure.
 896  * @param[in] attributes
 897  *   Pointer to device flow rule attributes.
 898  *
 899  * @return
 900  *   The value of lowest priority of flow.
 901  */
 902 uint32_t
 903 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 904                           const struct rte_flow_attr *attr)
 905 {
 906         struct mlx5_priv *priv = dev->data->dev_private;
 907
 908         if (!attr->group && !attr->transfer)
 909                 return priv->config.flow_prio - 2;
 910         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 911 }
 912
 913 /**
 914  * Calculate matcher priority of the flow.
 915  *
 916  * @param[in] dev
 917  *   Pointer to the Ethernet device structure.
 918  * @param[in] attr
 919  *   Pointer to device flow rule attributes.
 920  * @param[in] subpriority
 921  *   The priority based on the items.
 922  * @return
 923  *   The matcher priority of the flow.
 924  */
 925 uint16_t
 926 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 927                           const struct rte_flow_attr *attr,
 928                           uint32_t subpriority)
 929 {
 930         uint16_t priority = (uint16_t)attr->priority;
 931         struct mlx5_priv *priv = dev->data->dev_private;
 932
 933         if (!attr->group && !attr->transfer) {
 934                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 935                         priority = priv->config.flow_prio - 1;
 936                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 937         }
 938         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 939                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 940         return priority * 3 + subpriority;
 941 }
 942
 943 /**
 944  * Verify the @p item specifications (spec, last, mask) are compatible with the
 945  * NIC capabilities.
 946  *
 947  * @param[in] item
 948  *   Item specification.
 949  * @param[in] mask
 950  *   @p item->mask or flow default bit-masks.
 951  * @param[in] nic_mask
 952  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 953  * @param[in] size
 954  *   Bit-masks size in bytes.
 955  * @param[in] range_accepted
 956  *   True if range of values is accepted for specific fields, false otherwise.
 957  * @param[out] error
 958  *   Pointer to error structure.
 959  *
 960  * @return
 961  *   0 on success, a negative errno value otherwise and rte_errno is set.
 962  */
 963 int
 964 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 965                           const uint8_t *mask,
 966                           const uint8_t *nic_mask,
 967                           unsigned int size,
 968                           bool range_accepted,
 969                           struct rte_flow_error *error)
 970 {
 971         unsigned int i;
 972
 973         MLX5_ASSERT(nic_mask);
 974         for (i = 0; i < size; ++i)
 975                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 976                         return rte_flow_error_set(error, ENOTSUP,
 977                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 978                                                   item,
 979                                                   "mask enables non supported"
 980                                                   " bits");
 981         if (!item->spec && (item->mask || item->last))
 982                 return rte_flow_error_set(error, EINVAL,
 983                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 984                                           "mask/last without a spec is not"
 985                                           " supported");
 986         if (item->spec && item->last && !range_accepted) {
 987                 uint8_t spec[size];
 988                 uint8_t last[size];
 989                 unsigned int i;
 990                 int ret;
 991
 992                 for (i = 0; i < size; ++i) {
 993                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 994                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 995                 }
 996                 ret = memcmp(spec, last, size);
 997                 if (ret != 0)
 998                         return rte_flow_error_set(error, EINVAL,
 999                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1000                                                   item,
1001                                                   "range is not valid");
1002         }
1003         return 0;
1004 }
1005
1006 /**
1007  * Adjust the hash fields according to the @p flow information.
1008  *
1009  * @param[in] dev_flow.
1010  *   Pointer to the mlx5_flow.
1011  * @param[in] tunnel
1012  *   1 when the hash field is for a tunnel item.
1013  * @param[in] layer_types
1014  *   ETH_RSS_* types.
1015  * @param[in] hash_fields
1016  *   Item hash fields.
1017  *
1018  * @return
1019  *   The hash fields that should be used.
1020  */
1021 uint64_t
1022 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1023                             int tunnel __rte_unused, uint64_t layer_types,
1024                             uint64_t hash_fields)
1025 {
1026 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1027         int rss_request_inner = rss_desc->level >= 2;
1028
1029         /* Check RSS hash level for tunnel. */
1030         if (tunnel && rss_request_inner)
1031                 hash_fields |= IBV_RX_HASH_INNER;
1032         else if (tunnel || rss_request_inner)
1033                 return 0;
1034 #endif
1035         /* Check if requested layer matches RSS hash fields. */
1036         if (!(rss_desc->types & layer_types))
1037                 return 0;
1038         return hash_fields;
1039 }
1040
1041 /**
1042  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1043  * if several tunnel rules are used on this queue, the tunnel ptype will be
1044  * cleared.
1045  *
1046  * @param rxq_ctrl
1047  *   Rx queue to update.
1048  */
1049 static void
1050 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1051 {
1052         unsigned int i;
1053         uint32_t tunnel_ptype = 0;
1054
1055         /* Look up for the ptype to use. */
1056         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1057                 if (!rxq_ctrl->flow_tunnels_n[i])
1058                         continue;
1059                 if (!tunnel_ptype) {
1060                         tunnel_ptype = tunnels_info[i].ptype;
1061                 } else {
1062                         tunnel_ptype = 0;
1063                         break;
1064                 }
1065         }
1066         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1067 }
1068
1069 /**
1070  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1071  * flow.
1072  *
1073  * @param[in] dev
1074  *   Pointer to the Ethernet device structure.
1075  * @param[in] dev_handle
1076  *   Pointer to device flow handle structure.
1077  */
1078 void
1079 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1080                        struct mlx5_flow_handle *dev_handle)
1081 {
1082         struct mlx5_priv *priv = dev->data->dev_private;
1083         const int mark = dev_handle->mark;
1084         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1085         struct mlx5_ind_table_obj *ind_tbl = NULL;
1086         unsigned int i;
1087
1088         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1089                 struct mlx5_hrxq *hrxq;
1090
1091                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1092                               dev_handle->rix_hrxq);
1093                 if (hrxq)
1094                         ind_tbl = hrxq->ind_table;
1095         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1096                 struct mlx5_shared_action_rss *shared_rss;
1097
1098                 shared_rss = mlx5_ipool_get
1099                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1100                          dev_handle->rix_srss);
1101                 if (shared_rss)
1102                         ind_tbl = shared_rss->ind_tbl;
1103         }
1104         if (!ind_tbl)
1105                 return;
1106         for (i = 0; i != ind_tbl->queues_n; ++i) {
1107                 int idx = ind_tbl->queues[i];
1108                 struct mlx5_rxq_ctrl *rxq_ctrl =
1109                         container_of((*priv->rxqs)[idx],
1110                                      struct mlx5_rxq_ctrl, rxq);
1111
1112                 /*
1113                  * To support metadata register copy on Tx loopback,
1114                  * this must be always enabled (metadata may arive
1115                  * from other port - not from local flows only.
1116                  */
1117                 if (priv->config.dv_flow_en &&
1118                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1119                     mlx5_flow_ext_mreg_supported(dev)) {
1120                         rxq_ctrl->rxq.mark = 1;
1121                         rxq_ctrl->flow_mark_n = 1;
1122                 } else if (mark) {
1123                         rxq_ctrl->rxq.mark = 1;
1124                         rxq_ctrl->flow_mark_n++;
1125                 }
1126                 if (tunnel) {
1127                         unsigned int j;
1128
1129                         /* Increase the counter matching the flow. */
1130                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1131                                 if ((tunnels_info[j].tunnel &
1132                                      dev_handle->layers) ==
1133                                     tunnels_info[j].tunnel) {
1134                                         rxq_ctrl->flow_tunnels_n[j]++;
1135                                         break;
1136                                 }
1137                         }
1138                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1139                 }
1140         }
1141 }
1142
1143 /**
1144  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1145  *
1146  * @param[in] dev
1147  *   Pointer to the Ethernet device structure.
1148  * @param[in] flow
1149  *   Pointer to flow structure.
1150  */
1151 static void
1152 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         uint32_t handle_idx;
1156         struct mlx5_flow_handle *dev_handle;
1157
1158         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1159                        handle_idx, dev_handle, next)
1160                 flow_drv_rxq_flags_set(dev, dev_handle);
1161 }
1162
1163 /**
1164  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1165  * device flow if no other flow uses it with the same kind of request.
1166  *
1167  * @param dev
1168  *   Pointer to Ethernet device.
1169  * @param[in] dev_handle
1170  *   Pointer to the device flow handle structure.
1171  */
1172 static void
1173 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1174                         struct mlx5_flow_handle *dev_handle)
1175 {
1176         struct mlx5_priv *priv = dev->data->dev_private;
1177         const int mark = dev_handle->mark;
1178         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1179         struct mlx5_ind_table_obj *ind_tbl = NULL;
1180         unsigned int i;
1181
1182         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1183                 struct mlx5_hrxq *hrxq;
1184
1185                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1186                               dev_handle->rix_hrxq);
1187                 if (hrxq)
1188                         ind_tbl = hrxq->ind_table;
1189         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1190                 struct mlx5_shared_action_rss *shared_rss;
1191
1192                 shared_rss = mlx5_ipool_get
1193                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1194                          dev_handle->rix_srss);
1195                 if (shared_rss)
1196                         ind_tbl = shared_rss->ind_tbl;
1197         }
1198         if (!ind_tbl)
1199                 return;
1200         MLX5_ASSERT(dev->data->dev_started);
1201         for (i = 0; i != ind_tbl->queues_n; ++i) {
1202                 int idx = ind_tbl->queues[i];
1203                 struct mlx5_rxq_ctrl *rxq_ctrl =
1204                         container_of((*priv->rxqs)[idx],
1205                                      struct mlx5_rxq_ctrl, rxq);
1206
1207                 if (priv->config.dv_flow_en &&
1208                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1209                     mlx5_flow_ext_mreg_supported(dev)) {
1210                         rxq_ctrl->rxq.mark = 1;
1211                         rxq_ctrl->flow_mark_n = 1;
1212                 } else if (mark) {
1213                         rxq_ctrl->flow_mark_n--;
1214                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1215                 }
1216                 if (tunnel) {
1217                         unsigned int j;
1218
1219                         /* Decrease the counter matching the flow. */
1220                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1221                                 if ((tunnels_info[j].tunnel &
1222                                      dev_handle->layers) ==
1223                                     tunnels_info[j].tunnel) {
1224                                         rxq_ctrl->flow_tunnels_n[j]--;
1225                                         break;
1226                                 }
1227                         }
1228                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1229                 }
1230         }
1231 }
1232
1233 /**
1234  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1235  * @p flow if no other flow uses it with the same kind of request.
1236  *
1237  * @param dev
1238  *   Pointer to Ethernet device.
1239  * @param[in] flow
1240  *   Pointer to the flow.
1241  */
1242 static void
1243 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1244 {
1245         struct mlx5_priv *priv = dev->data->dev_private;
1246         uint32_t handle_idx;
1247         struct mlx5_flow_handle *dev_handle;
1248
1249         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1250                        handle_idx, dev_handle, next)
1251                 flow_drv_rxq_flags_trim(dev, dev_handle);
1252 }
1253
1254 /**
1255  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1256  *
1257  * @param dev
1258  *   Pointer to Ethernet device.
1259  */
1260 static void
1261 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1262 {
1263         struct mlx5_priv *priv = dev->data->dev_private;
1264         unsigned int i;
1265
1266         for (i = 0; i != priv->rxqs_n; ++i) {
1267                 struct mlx5_rxq_ctrl *rxq_ctrl;
1268                 unsigned int j;
1269
1270                 if (!(*priv->rxqs)[i])
1271                         continue;
1272                 rxq_ctrl = container_of((*priv->rxqs)[i],
1273                                         struct mlx5_rxq_ctrl, rxq);
1274                 rxq_ctrl->flow_mark_n = 0;
1275                 rxq_ctrl->rxq.mark = 0;
1276                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1277                         rxq_ctrl->flow_tunnels_n[j] = 0;
1278                 rxq_ctrl->rxq.tunnel = 0;
1279         }
1280 }
1281
1282 /**
1283  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1284  *
1285  * @param[in] dev
1286  *   Pointer to the Ethernet device structure.
1287  */
1288 void
1289 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1290 {
1291         struct mlx5_priv *priv = dev->data->dev_private;
1292         struct mlx5_rxq_data *data;
1293         unsigned int i;
1294
1295         for (i = 0; i != priv->rxqs_n; ++i) {
1296                 if (!(*priv->rxqs)[i])
1297                         continue;
1298                 data = (*priv->rxqs)[i];
1299                 if (!rte_flow_dynf_metadata_avail()) {
1300                         data->dynf_meta = 0;
1301                         data->flow_meta_mask = 0;
1302                         data->flow_meta_offset = -1;
1303                         data->flow_meta_port_mask = 0;
1304                 } else {
1305                         data->dynf_meta = 1;
1306                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1307                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1308                         data->flow_meta_port_mask = (uint32_t)~0;
1309                         if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16)
1310                                 data->flow_meta_port_mask >>= 16;
1311                 }
1312         }
1313 }
1314
1315 /*
1316  * return a pointer to the desired action in the list of actions.
1317  *
1318  * @param[in] actions
1319  *   The list of actions to search the action in.
1320  * @param[in] action
1321  *   The action to find.
1322  *
1323  * @return
1324  *   Pointer to the action in the list, if found. NULL otherwise.
1325  */
1326 const struct rte_flow_action *
1327 mlx5_flow_find_action(const struct rte_flow_action *actions,
1328                       enum rte_flow_action_type action)
1329 {
1330         if (actions == NULL)
1331                 return NULL;
1332         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1333                 if (actions->type == action)
1334                         return actions;
1335         return NULL;
1336 }
1337
1338 /*
1339  * Validate the flag action.
1340  *
1341  * @param[in] action_flags
1342  *   Bit-fields that holds the actions detected until now.
1343  * @param[in] attr
1344  *   Attributes of flow that includes this action.
1345  * @param[out] error
1346  *   Pointer to error structure.
1347  *
1348  * @return
1349  *   0 on success, a negative errno value otherwise and rte_errno is set.
1350  */
1351 int
1352 mlx5_flow_validate_action_flag(uint64_t action_flags,
1353                                const struct rte_flow_attr *attr,
1354                                struct rte_flow_error *error)
1355 {
1356         if (action_flags & MLX5_FLOW_ACTION_MARK)
1357                 return rte_flow_error_set(error, EINVAL,
1358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1359                                           "can't mark and flag in same flow");
1360         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1361                 return rte_flow_error_set(error, EINVAL,
1362                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1363                                           "can't have 2 flag"
1364                                           " actions in same flow");
1365         if (attr->egress)
1366                 return rte_flow_error_set(error, ENOTSUP,
1367                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1368                                           "flag action not supported for "
1369                                           "egress");
1370         return 0;
1371 }
1372
1373 /*
1374  * Validate the mark action.
1375  *
1376  * @param[in] action
1377  *   Pointer to the queue action.
1378  * @param[in] action_flags
1379  *   Bit-fields that holds the actions detected until now.
1380  * @param[in] attr
1381  *   Attributes of flow that includes this action.
1382  * @param[out] error
1383  *   Pointer to error structure.
1384  *
1385  * @return
1386  *   0 on success, a negative errno value otherwise and rte_errno is set.
1387  */
1388 int
1389 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1390                                uint64_t action_flags,
1391                                const struct rte_flow_attr *attr,
1392                                struct rte_flow_error *error)
1393 {
1394         const struct rte_flow_action_mark *mark = action->conf;
1395
1396         if (!mark)
1397                 return rte_flow_error_set(error, EINVAL,
1398                                           RTE_FLOW_ERROR_TYPE_ACTION,
1399                                           action,
1400                                           "configuration cannot be null");
1401         if (mark->id >= MLX5_FLOW_MARK_MAX)
1402                 return rte_flow_error_set(error, EINVAL,
1403                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1404                                           &mark->id,
1405                                           "mark id must in 0 <= id < "
1406                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1407         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1408                 return rte_flow_error_set(error, EINVAL,
1409                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1410                                           "can't flag and mark in same flow");
1411         if (action_flags & MLX5_FLOW_ACTION_MARK)
1412                 return rte_flow_error_set(error, EINVAL,
1413                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1414                                           "can't have 2 mark actions in same"
1415                                           " flow");
1416         if (attr->egress)
1417                 return rte_flow_error_set(error, ENOTSUP,
1418                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1419                                           "mark action not supported for "
1420                                           "egress");
1421         return 0;
1422 }
1423
1424 /*
1425  * Validate the drop action.
1426  *
1427  * @param[in] action_flags
1428  *   Bit-fields that holds the actions detected until now.
1429  * @param[in] attr
1430  *   Attributes of flow that includes this action.
1431  * @param[out] error
1432  *   Pointer to error structure.
1433  *
1434  * @return
1435  *   0 on success, a negative errno value otherwise and rte_errno is set.
1436  */
1437 int
1438 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1439                                const struct rte_flow_attr *attr,
1440                                struct rte_flow_error *error)
1441 {
1442         if (attr->egress)
1443                 return rte_flow_error_set(error, ENOTSUP,
1444                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1445                                           "drop action not supported for "
1446                                           "egress");
1447         return 0;
1448 }
1449
1450 /*
1451  * Validate the queue action.
1452  *
1453  * @param[in] action
1454  *   Pointer to the queue action.
1455  * @param[in] action_flags
1456  *   Bit-fields that holds the actions detected until now.
1457  * @param[in] dev
1458  *   Pointer to the Ethernet device structure.
1459  * @param[in] attr
1460  *   Attributes of flow that includes this action.
1461  * @param[out] error
1462  *   Pointer to error structure.
1463  *
1464  * @return
1465  *   0 on success, a negative errno value otherwise and rte_errno is set.
1466  */
1467 int
1468 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1469                                 uint64_t action_flags,
1470                                 struct rte_eth_dev *dev,
1471                                 const struct rte_flow_attr *attr,
1472                                 struct rte_flow_error *error)
1473 {
1474         struct mlx5_priv *priv = dev->data->dev_private;
1475         const struct rte_flow_action_queue *queue = action->conf;
1476
1477         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1478                 return rte_flow_error_set(error, EINVAL,
1479                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1480                                           "can't have 2 fate actions in"
1481                                           " same flow");
1482         if (!priv->rxqs_n)
1483                 return rte_flow_error_set(error, EINVAL,
1484                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1485                                           NULL, "No Rx queues configured");
1486         if (queue->index >= priv->rxqs_n)
1487                 return rte_flow_error_set(error, EINVAL,
1488                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1489                                           &queue->index,
1490                                           "queue index out of range");
1491         if (!(*priv->rxqs)[queue->index])
1492                 return rte_flow_error_set(error, EINVAL,
1493                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1494                                           &queue->index,
1495                                           "queue is not configured");
1496         if (attr->egress)
1497                 return rte_flow_error_set(error, ENOTSUP,
1498                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1499                                           "queue action not supported for "
1500                                           "egress");
1501         return 0;
1502 }
1503
1504 /*
1505  * Validate the rss action.
1506  *
1507  * @param[in] dev
1508  *   Pointer to the Ethernet device structure.
1509  * @param[in] action
1510  *   Pointer to the queue action.
1511  * @param[out] error
1512  *   Pointer to error structure.
1513  *
1514  * @return
1515  *   0 on success, a negative errno value otherwise and rte_errno is set.
1516  */
1517 int
1518 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1519                          const struct rte_flow_action *action,
1520                          struct rte_flow_error *error)
1521 {
1522         struct mlx5_priv *priv = dev->data->dev_private;
1523         const struct rte_flow_action_rss *rss = action->conf;
1524         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1525         unsigned int i;
1526
1527         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1528             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1529                 return rte_flow_error_set(error, ENOTSUP,
1530                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1531                                           &rss->func,
1532                                           "RSS hash function not supported");
1533 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1534         if (rss->level > 2)
1535 #else
1536         if (rss->level > 1)
1537 #endif
1538                 return rte_flow_error_set(error, ENOTSUP,
1539                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1540                                           &rss->level,
1541                                           "tunnel RSS is not supported");
1542         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1543         if (rss->key_len == 0 && rss->key != NULL)
1544                 return rte_flow_error_set(error, ENOTSUP,
1545                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1546                                           &rss->key_len,
1547                                           "RSS hash key length 0");
1548         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1549                 return rte_flow_error_set(error, ENOTSUP,
1550                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1551                                           &rss->key_len,
1552                                           "RSS hash key too small");
1553         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1554                 return rte_flow_error_set(error, ENOTSUP,
1555                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1556                                           &rss->key_len,
1557                                           "RSS hash key too large");
1558         if (rss->queue_num > priv->config.ind_table_max_size)
1559                 return rte_flow_error_set(error, ENOTSUP,
1560                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1561                                           &rss->queue_num,
1562                                           "number of queues too large");
1563         if (rss->types & MLX5_RSS_HF_MASK)
1564                 return rte_flow_error_set(error, ENOTSUP,
1565                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1566                                           &rss->types,
1567                                           "some RSS protocols are not"
1568                                           " supported");
1569         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1570             !(rss->types & ETH_RSS_IP))
1571                 return rte_flow_error_set(error, EINVAL,
1572                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1573                                           "L3 partial RSS requested but L3 RSS"
1574                                           " type not specified");
1575         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1576             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1577                 return rte_flow_error_set(error, EINVAL,
1578                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1579                                           "L4 partial RSS requested but L4 RSS"
1580                                           " type not specified");
1581         if (!priv->rxqs_n)
1582                 return rte_flow_error_set(error, EINVAL,
1583                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1584                                           NULL, "No Rx queues configured");
1585         if (!rss->queue_num)
1586                 return rte_flow_error_set(error, EINVAL,
1587                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1588                                           NULL, "No queues configured");
1589         for (i = 0; i != rss->queue_num; ++i) {
1590                 struct mlx5_rxq_ctrl *rxq_ctrl;
1591
1592                 if (rss->queue[i] >= priv->rxqs_n)
1593                         return rte_flow_error_set
1594                                 (error, EINVAL,
1595                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1596                                  &rss->queue[i], "queue index out of range");
1597                 if (!(*priv->rxqs)[rss->queue[i]])
1598                         return rte_flow_error_set
1599                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1600                                  &rss->queue[i], "queue is not configured");
1601                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1602                                         struct mlx5_rxq_ctrl, rxq);
1603                 if (i == 0)
1604                         rxq_type = rxq_ctrl->type;
1605                 if (rxq_type != rxq_ctrl->type)
1606                         return rte_flow_error_set
1607                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1608                                  &rss->queue[i],
1609                                  "combining hairpin and regular RSS queues is not supported");
1610         }
1611         return 0;
1612 }
1613
1614 /*
1615  * Validate the rss action.
1616  *
1617  * @param[in] action
1618  *   Pointer to the queue action.
1619  * @param[in] action_flags
1620  *   Bit-fields that holds the actions detected until now.
1621  * @param[in] dev
1622  *   Pointer to the Ethernet device structure.
1623  * @param[in] attr
1624  *   Attributes of flow that includes this action.
1625  * @param[in] item_flags
1626  *   Items that were detected.
1627  * @param[out] error
1628  *   Pointer to error structure.
1629  *
1630  * @return
1631  *   0 on success, a negative errno value otherwise and rte_errno is set.
1632  */
1633 int
1634 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1635                               uint64_t action_flags,
1636                               struct rte_eth_dev *dev,
1637                               const struct rte_flow_attr *attr,
1638                               uint64_t item_flags,
1639                               struct rte_flow_error *error)
1640 {
1641         const struct rte_flow_action_rss *rss = action->conf;
1642         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1643         int ret;
1644
1645         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1646                 return rte_flow_error_set(error, EINVAL,
1647                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1648                                           "can't have 2 fate actions"
1649                                           " in same flow");
1650         ret = mlx5_validate_action_rss(dev, action, error);
1651         if (ret)
1652                 return ret;
1653         if (attr->egress)
1654                 return rte_flow_error_set(error, ENOTSUP,
1655                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1656                                           "rss action not supported for "
1657                                           "egress");
1658         if (rss->level > 1 && !tunnel)
1659                 return rte_flow_error_set(error, EINVAL,
1660                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1661                                           "inner RSS is not supported for "
1662                                           "non-tunnel flows");
1663         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1664             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1665                 return rte_flow_error_set(error, EINVAL,
1666                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1667                                           "RSS on eCPRI is not supported now");
1668         }
1669         return 0;
1670 }
1671
1672 /*
1673  * Validate the default miss action.
1674  *
1675  * @param[in] action_flags
1676  *   Bit-fields that holds the actions detected until now.
1677  * @param[out] error
1678  *   Pointer to error structure.
1679  *
1680  * @return
1681  *   0 on success, a negative errno value otherwise and rte_errno is set.
1682  */
1683 int
1684 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1685                                 const struct rte_flow_attr *attr,
1686                                 struct rte_flow_error *error)
1687 {
1688         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1689                 return rte_flow_error_set(error, EINVAL,
1690                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1691                                           "can't have 2 fate actions in"
1692                                           " same flow");
1693         if (attr->egress)
1694                 return rte_flow_error_set(error, ENOTSUP,
1695                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1696                                           "default miss action not supported "
1697                                           "for egress");
1698         if (attr->group)
1699                 return rte_flow_error_set(error, ENOTSUP,
1700                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1701                                           "only group 0 is supported");
1702         if (attr->transfer)
1703                 return rte_flow_error_set(error, ENOTSUP,
1704                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1705                                           NULL, "transfer is not supported");
1706         return 0;
1707 }
1708
1709 /*
1710  * Validate the count action.
1711  *
1712  * @param[in] dev
1713  *   Pointer to the Ethernet device structure.
1714  * @param[in] attr
1715  *   Attributes of flow that includes this action.
1716  * @param[out] error
1717  *   Pointer to error structure.
1718  *
1719  * @return
1720  *   0 on success, a negative errno value otherwise and rte_errno is set.
1721  */
1722 int
1723 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1724                                 const struct rte_flow_attr *attr,
1725                                 struct rte_flow_error *error)
1726 {
1727         if (attr->egress)
1728                 return rte_flow_error_set(error, ENOTSUP,
1729                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1730                                           "count action not supported for "
1731                                           "egress");
1732         return 0;
1733 }
1734
1735 /*
1736  * Validate the ASO CT action.
1737  *
1738  * @param[in] dev
1739  *   Pointer to the Ethernet device structure.
1740  * @param[in] conntrack
1741  *   Pointer to the CT action profile.
1742  * @param[out] error
1743  *   Pointer to error structure.
1744  *
1745  * @return
1746  *   0 on success, a negative errno value otherwise and rte_errno is set.
1747  */
1748 int
1749 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1750                         const struct rte_flow_action_conntrack *conntrack,
1751                         struct rte_flow_error *error)
1752 {
1753         RTE_SET_USED(dev);
1754
1755         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1756                 return rte_flow_error_set(error, EINVAL,
1757                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1758                                           "Invalid CT state");
1759         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1760                 return rte_flow_error_set(error, EINVAL,
1761                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1762                                           "Invalid last TCP packet flag");
1763         return 0;
1764 }
1765
1766 /**
1767  * Verify the @p attributes will be correctly understood by the NIC and store
1768  * them in the @p flow if everything is correct.
1769  *
1770  * @param[in] dev
1771  *   Pointer to the Ethernet device structure.
1772  * @param[in] attributes
1773  *   Pointer to flow attributes
1774  * @param[out] error
1775  *   Pointer to error structure.
1776  *
1777  * @return
1778  *   0 on success, a negative errno value otherwise and rte_errno is set.
1779  */
1780 int
1781 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1782                               const struct rte_flow_attr *attributes,
1783                               struct rte_flow_error *error)
1784 {
1785         struct mlx5_priv *priv = dev->data->dev_private;
1786         uint32_t priority_max = priv->config.flow_prio - 1;
1787
1788         if (attributes->group)
1789                 return rte_flow_error_set(error, ENOTSUP,
1790                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1791                                           NULL, "groups is not supported");
1792         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1793             attributes->priority >= priority_max)
1794                 return rte_flow_error_set(error, ENOTSUP,
1795                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1796                                           NULL, "priority out of range");
1797         if (attributes->egress)
1798                 return rte_flow_error_set(error, ENOTSUP,
1799                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1800                                           "egress is not supported");
1801         if (attributes->transfer && !priv->config.dv_esw_en)
1802                 return rte_flow_error_set(error, ENOTSUP,
1803                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1804                                           NULL, "transfer is not supported");
1805         if (!attributes->ingress)
1806                 return rte_flow_error_set(error, EINVAL,
1807                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1808                                           NULL,
1809                                           "ingress attribute is mandatory");
1810         return 0;
1811 }
1812
1813 /**
1814  * Validate ICMP6 item.
1815  *
1816  * @param[in] item
1817  *   Item specification.
1818  * @param[in] item_flags
1819  *   Bit-fields that holds the items detected until now.
1820  * @param[in] ext_vlan_sup
1821  *   Whether extended VLAN features are supported or not.
1822  * @param[out] error
1823  *   Pointer to error structure.
1824  *
1825  * @return
1826  *   0 on success, a negative errno value otherwise and rte_errno is set.
1827  */
1828 int
1829 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1830                                uint64_t item_flags,
1831                                uint8_t target_protocol,
1832                                struct rte_flow_error *error)
1833 {
1834         const struct rte_flow_item_icmp6 *mask = item->mask;
1835         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1836         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1837                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1838         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1839                                       MLX5_FLOW_LAYER_OUTER_L4;
1840         int ret;
1841
1842         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1843                 return rte_flow_error_set(error, EINVAL,
1844                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1845                                           "protocol filtering not compatible"
1846                                           " with ICMP6 layer");
1847         if (!(item_flags & l3m))
1848                 return rte_flow_error_set(error, EINVAL,
1849                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1850                                           "IPv6 is mandatory to filter on"
1851                                           " ICMP6");
1852         if (item_flags & l4m)
1853                 return rte_flow_error_set(error, EINVAL,
1854                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1855                                           "multiple L4 layers not supported");
1856         if (!mask)
1857                 mask = &rte_flow_item_icmp6_mask;
1858         ret = mlx5_flow_item_acceptable
1859                 (item, (const uint8_t *)mask,
1860                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1861                  sizeof(struct rte_flow_item_icmp6),
1862                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1863         if (ret < 0)
1864                 return ret;
1865         return 0;
1866 }
1867
1868 /**
1869  * Validate ICMP item.
1870  *
1871  * @param[in] item
1872  *   Item specification.
1873  * @param[in] item_flags
1874  *   Bit-fields that holds the items detected until now.
1875  * @param[out] error
1876  *   Pointer to error structure.
1877  *
1878  * @return
1879  *   0 on success, a negative errno value otherwise and rte_errno is set.
1880  */
1881 int
1882 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1883                              uint64_t item_flags,
1884                              uint8_t target_protocol,
1885                              struct rte_flow_error *error)
1886 {
1887         const struct rte_flow_item_icmp *mask = item->mask;
1888         const struct rte_flow_item_icmp nic_mask = {
1889                 .hdr.icmp_type = 0xff,
1890                 .hdr.icmp_code = 0xff,
1891                 .hdr.icmp_ident = RTE_BE16(0xffff),
1892                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1893         };
1894         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1895         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1896                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1897         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1898                                       MLX5_FLOW_LAYER_OUTER_L4;
1899         int ret;
1900
1901         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1902                 return rte_flow_error_set(error, EINVAL,
1903                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1904                                           "protocol filtering not compatible"
1905                                           " with ICMP layer");
1906         if (!(item_flags & l3m))
1907                 return rte_flow_error_set(error, EINVAL,
1908                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1909                                           "IPv4 is mandatory to filter"
1910                                           " on ICMP");
1911         if (item_flags & l4m)
1912                 return rte_flow_error_set(error, EINVAL,
1913                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1914                                           "multiple L4 layers not supported");
1915         if (!mask)
1916                 mask = &nic_mask;
1917         ret = mlx5_flow_item_acceptable
1918                 (item, (const uint8_t *)mask,
1919                  (const uint8_t *)&nic_mask,
1920                  sizeof(struct rte_flow_item_icmp),
1921                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1922         if (ret < 0)
1923                 return ret;
1924         return 0;
1925 }
1926
1927 /**
1928  * Validate Ethernet item.
1929  *
1930  * @param[in] item
1931  *   Item specification.
1932  * @param[in] item_flags
1933  *   Bit-fields that holds the items detected until now.
1934  * @param[out] error
1935  *   Pointer to error structure.
1936  *
1937  * @return
1938  *   0 on success, a negative errno value otherwise and rte_errno is set.
1939  */
1940 int
1941 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1942                             uint64_t item_flags, bool ext_vlan_sup,
1943                             struct rte_flow_error *error)
1944 {
1945         const struct rte_flow_item_eth *mask = item->mask;
1946         const struct rte_flow_item_eth nic_mask = {
1947                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1948                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1949                 .type = RTE_BE16(0xffff),
1950                 .has_vlan = ext_vlan_sup ? 1 : 0,
1951         };
1952         int ret;
1953         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1954         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1955                                        MLX5_FLOW_LAYER_OUTER_L2;
1956
1957         if (item_flags & ethm)
1958                 return rte_flow_error_set(error, ENOTSUP,
1959                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1960                                           "multiple L2 layers not supported");
1961         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1962             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1963                 return rte_flow_error_set(error, EINVAL,
1964                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1965                                           "L2 layer should not follow "
1966                                           "L3 layers");
1967         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1968             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1969                 return rte_flow_error_set(error, EINVAL,
1970                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1971                                           "L2 layer should not follow VLAN");
1972         if (!mask)
1973                 mask = &rte_flow_item_eth_mask;
1974         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1975                                         (const uint8_t *)&nic_mask,
1976                                         sizeof(struct rte_flow_item_eth),
1977                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1978         return ret;
1979 }
1980
1981 /**
1982  * Validate VLAN item.
1983  *
1984  * @param[in] item
1985  *   Item specification.
1986  * @param[in] item_flags
1987  *   Bit-fields that holds the items detected until now.
1988  * @param[in] dev
1989  *   Ethernet device flow is being created on.
1990  * @param[out] error
1991  *   Pointer to error structure.
1992  *
1993  * @return
1994  *   0 on success, a negative errno value otherwise and rte_errno is set.
1995  */
1996 int
1997 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1998                              uint64_t item_flags,
1999                              struct rte_eth_dev *dev,
2000                              struct rte_flow_error *error)
2001 {
2002         const struct rte_flow_item_vlan *spec = item->spec;
2003         const struct rte_flow_item_vlan *mask = item->mask;
2004         const struct rte_flow_item_vlan nic_mask = {
2005                 .tci = RTE_BE16(UINT16_MAX),
2006                 .inner_type = RTE_BE16(UINT16_MAX),
2007         };
2008         uint16_t vlan_tag = 0;
2009         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2010         int ret;
2011         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2012                                         MLX5_FLOW_LAYER_INNER_L4) :
2013                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2014                                         MLX5_FLOW_LAYER_OUTER_L4);
2015         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2016                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2017
2018         if (item_flags & vlanm)
2019                 return rte_flow_error_set(error, EINVAL,
2020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2021                                           "multiple VLAN layers not supported");
2022         else if ((item_flags & l34m) != 0)
2023                 return rte_flow_error_set(error, EINVAL,
2024                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2025                                           "VLAN cannot follow L3/L4 layer");
2026         if (!mask)
2027                 mask = &rte_flow_item_vlan_mask;
2028         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2029                                         (const uint8_t *)&nic_mask,
2030                                         sizeof(struct rte_flow_item_vlan),
2031                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2032         if (ret)
2033                 return ret;
2034         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2035                 struct mlx5_priv *priv = dev->data->dev_private;
2036
2037                 if (priv->vmwa_context) {
2038                         /*
2039                          * Non-NULL context means we have a virtual machine
2040                          * and SR-IOV enabled, we have to create VLAN interface
2041                          * to make hypervisor to setup E-Switch vport
2042                          * context correctly. We avoid creating the multiple
2043                          * VLAN interfaces, so we cannot support VLAN tag mask.
2044                          */
2045                         return rte_flow_error_set(error, EINVAL,
2046                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2047                                                   item,
2048                                                   "VLAN tag mask is not"
2049                                                   " supported in virtual"
2050                                                   " environment");
2051                 }
2052         }
2053         if (spec) {
2054                 vlan_tag = spec->tci;
2055                 vlan_tag &= mask->tci;
2056         }
2057         /*
2058          * From verbs perspective an empty VLAN is equivalent
2059          * to a packet without VLAN layer.
2060          */
2061         if (!vlan_tag)
2062                 return rte_flow_error_set(error, EINVAL,
2063                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2064                                           item->spec,
2065                                           "VLAN cannot be empty");
2066         return 0;
2067 }
2068
2069 /**
2070  * Validate IPV4 item.
2071  *
2072  * @param[in] item
2073  *   Item specification.
2074  * @param[in] item_flags
2075  *   Bit-fields that holds the items detected until now.
2076  * @param[in] last_item
2077  *   Previous validated item in the pattern items.
2078  * @param[in] ether_type
2079  *   Type in the ethernet layer header (including dot1q).
2080  * @param[in] acc_mask
2081  *   Acceptable mask, if NULL default internal default mask
2082  *   will be used to check whether item fields are supported.
2083  * @param[in] range_accepted
2084  *   True if range of values is accepted for specific fields, false otherwise.
2085  * @param[out] error
2086  *   Pointer to error structure.
2087  *
2088  * @return
2089  *   0 on success, a negative errno value otherwise and rte_errno is set.
2090  */
2091 int
2092 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2093                              uint64_t item_flags,
2094                              uint64_t last_item,
2095                              uint16_t ether_type,
2096                              const struct rte_flow_item_ipv4 *acc_mask,
2097                              bool range_accepted,
2098                              struct rte_flow_error *error)
2099 {
2100         const struct rte_flow_item_ipv4 *mask = item->mask;
2101         const struct rte_flow_item_ipv4 *spec = item->spec;
2102         const struct rte_flow_item_ipv4 nic_mask = {
2103                 .hdr = {
2104                         .src_addr = RTE_BE32(0xffffffff),
2105                         .dst_addr = RTE_BE32(0xffffffff),
2106                         .type_of_service = 0xff,
2107                         .next_proto_id = 0xff,
2108                 },
2109         };
2110         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2111         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2112                                       MLX5_FLOW_LAYER_OUTER_L3;
2113         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2114                                       MLX5_FLOW_LAYER_OUTER_L4;
2115         int ret;
2116         uint8_t next_proto = 0xFF;
2117         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2118                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2119                                   MLX5_FLOW_LAYER_INNER_VLAN);
2120
2121         if ((last_item & l2_vlan) && ether_type &&
2122             ether_type != RTE_ETHER_TYPE_IPV4)
2123                 return rte_flow_error_set(error, EINVAL,
2124                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2125                                           "IPv4 cannot follow L2/VLAN layer "
2126                                           "which ether type is not IPv4");
2127         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2128                 if (mask && spec)
2129                         next_proto = mask->hdr.next_proto_id &
2130                                      spec->hdr.next_proto_id;
2131                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2132                         return rte_flow_error_set(error, EINVAL,
2133                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2134                                                   item,
2135                                                   "multiple tunnel "
2136                                                   "not supported");
2137         }
2138         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2139                 return rte_flow_error_set(error, EINVAL,
2140                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2141                                           "wrong tunnel type - IPv6 specified "
2142                                           "but IPv4 item provided");
2143         if (item_flags & l3m)
2144                 return rte_flow_error_set(error, ENOTSUP,
2145                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2146                                           "multiple L3 layers not supported");
2147         else if (item_flags & l4m)
2148                 return rte_flow_error_set(error, EINVAL,
2149                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2150                                           "L3 cannot follow an L4 layer.");
2151         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2152                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2153                 return rte_flow_error_set(error, EINVAL,
2154                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2155                                           "L3 cannot follow an NVGRE layer.");
2156         if (!mask)
2157                 mask = &rte_flow_item_ipv4_mask;
2158         else if (mask->hdr.next_proto_id != 0 &&
2159                  mask->hdr.next_proto_id != 0xff)
2160                 return rte_flow_error_set(error, EINVAL,
2161                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2162                                           "partial mask is not supported"
2163                                           " for protocol");
2164         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2165                                         acc_mask ? (const uint8_t *)acc_mask
2166                                                  : (const uint8_t *)&nic_mask,
2167                                         sizeof(struct rte_flow_item_ipv4),
2168                                         range_accepted, error);
2169         if (ret < 0)
2170                 return ret;
2171         return 0;
2172 }
2173
2174 /**
2175  * Validate IPV6 item.
2176  *
2177  * @param[in] item
2178  *   Item specification.
2179  * @param[in] item_flags
2180  *   Bit-fields that holds the items detected until now.
2181  * @param[in] last_item
2182  *   Previous validated item in the pattern items.
2183  * @param[in] ether_type
2184  *   Type in the ethernet layer header (including dot1q).
2185  * @param[in] acc_mask
2186  *   Acceptable mask, if NULL default internal default mask
2187  *   will be used to check whether item fields are supported.
2188  * @param[out] error
2189  *   Pointer to error structure.
2190  *
2191  * @return
2192  *   0 on success, a negative errno value otherwise and rte_errno is set.
2193  */
2194 int
2195 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2196                              uint64_t item_flags,
2197                              uint64_t last_item,
2198                              uint16_t ether_type,
2199                              const struct rte_flow_item_ipv6 *acc_mask,
2200                              struct rte_flow_error *error)
2201 {
2202         const struct rte_flow_item_ipv6 *mask = item->mask;
2203         const struct rte_flow_item_ipv6 *spec = item->spec;
2204         const struct rte_flow_item_ipv6 nic_mask = {
2205                 .hdr = {
2206                         .src_addr =
2207                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2208                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2209                         .dst_addr =
2210                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2211                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2212                         .vtc_flow = RTE_BE32(0xffffffff),
2213                         .proto = 0xff,
2214                 },
2215         };
2216         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2217         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2218                                       MLX5_FLOW_LAYER_OUTER_L3;
2219         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2220                                       MLX5_FLOW_LAYER_OUTER_L4;
2221         int ret;
2222         uint8_t next_proto = 0xFF;
2223         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2224                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2225                                   MLX5_FLOW_LAYER_INNER_VLAN);
2226
2227         if ((last_item & l2_vlan) && ether_type &&
2228             ether_type != RTE_ETHER_TYPE_IPV6)
2229                 return rte_flow_error_set(error, EINVAL,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2231                                           "IPv6 cannot follow L2/VLAN layer "
2232                                           "which ether type is not IPv6");
2233         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2234                 next_proto = spec->hdr.proto;
2235         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2236                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2237                         return rte_flow_error_set(error, EINVAL,
2238                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2239                                                   item,
2240                                                   "multiple tunnel "
2241                                                   "not supported");
2242         }
2243         if (next_proto == IPPROTO_HOPOPTS  ||
2244             next_proto == IPPROTO_ROUTING  ||
2245             next_proto == IPPROTO_FRAGMENT ||
2246             next_proto == IPPROTO_ESP      ||
2247             next_proto == IPPROTO_AH       ||
2248             next_proto == IPPROTO_DSTOPTS)
2249                 return rte_flow_error_set(error, EINVAL,
2250                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2251                                           "IPv6 proto (next header) should "
2252                                           "not be set as extension header");
2253         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2254                 return rte_flow_error_set(error, EINVAL,
2255                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2256                                           "wrong tunnel type - IPv4 specified "
2257                                           "but IPv6 item provided");
2258         if (item_flags & l3m)
2259                 return rte_flow_error_set(error, ENOTSUP,
2260                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2261                                           "multiple L3 layers not supported");
2262         else if (item_flags & l4m)
2263                 return rte_flow_error_set(error, EINVAL,
2264                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2265                                           "L3 cannot follow an L4 layer.");
2266         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2267                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2268                 return rte_flow_error_set(error, EINVAL,
2269                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2270                                           "L3 cannot follow an NVGRE layer.");
2271         if (!mask)
2272                 mask = &rte_flow_item_ipv6_mask;
2273         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2274                                         acc_mask ? (const uint8_t *)acc_mask
2275                                                  : (const uint8_t *)&nic_mask,
2276                                         sizeof(struct rte_flow_item_ipv6),
2277                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2278         if (ret < 0)
2279                 return ret;
2280         return 0;
2281 }
2282
2283 /**
2284  * Validate UDP item.
2285  *
2286  * @param[in] item
2287  *   Item specification.
2288  * @param[in] item_flags
2289  *   Bit-fields that holds the items detected until now.
2290  * @param[in] target_protocol
2291  *   The next protocol in the previous item.
2292  * @param[in] flow_mask
2293  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2294  * @param[out] error
2295  *   Pointer to error structure.
2296  *
2297  * @return
2298  *   0 on success, a negative errno value otherwise and rte_errno is set.
2299  */
2300 int
2301 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2302                             uint64_t item_flags,
2303                             uint8_t target_protocol,
2304                             struct rte_flow_error *error)
2305 {
2306         const struct rte_flow_item_udp *mask = item->mask;
2307         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2308         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2309                                       MLX5_FLOW_LAYER_OUTER_L3;
2310         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2311                                       MLX5_FLOW_LAYER_OUTER_L4;
2312         int ret;
2313
2314         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2315                 return rte_flow_error_set(error, EINVAL,
2316                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2317                                           "protocol filtering not compatible"
2318                                           " with UDP layer");
2319         if (!(item_flags & l3m))
2320                 return rte_flow_error_set(error, EINVAL,
2321                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2322                                           "L3 is mandatory to filter on L4");
2323         if (item_flags & l4m)
2324                 return rte_flow_error_set(error, EINVAL,
2325                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2326                                           "multiple L4 layers not supported");
2327         if (!mask)
2328                 mask = &rte_flow_item_udp_mask;
2329         ret = mlx5_flow_item_acceptable
2330                 (item, (const uint8_t *)mask,
2331                  (const uint8_t *)&rte_flow_item_udp_mask,
2332                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2333                  error);
2334         if (ret < 0)
2335                 return ret;
2336         return 0;
2337 }
2338
2339 /**
2340  * Validate TCP item.
2341  *
2342  * @param[in] item
2343  *   Item specification.
2344  * @param[in] item_flags
2345  *   Bit-fields that holds the items detected until now.
2346  * @param[in] target_protocol
2347  *   The next protocol in the previous item.
2348  * @param[out] error
2349  *   Pointer to error structure.
2350  *
2351  * @return
2352  *   0 on success, a negative errno value otherwise and rte_errno is set.
2353  */
2354 int
2355 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2356                             uint64_t item_flags,
2357                             uint8_t target_protocol,
2358                             const struct rte_flow_item_tcp *flow_mask,
2359                             struct rte_flow_error *error)
2360 {
2361         const struct rte_flow_item_tcp *mask = item->mask;
2362         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2363         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2364                                       MLX5_FLOW_LAYER_OUTER_L3;
2365         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2366                                       MLX5_FLOW_LAYER_OUTER_L4;
2367         int ret;
2368
2369         MLX5_ASSERT(flow_mask);
2370         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2373                                           "protocol filtering not compatible"
2374                                           " with TCP layer");
2375         if (!(item_flags & l3m))
2376                 return rte_flow_error_set(error, EINVAL,
2377                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2378                                           "L3 is mandatory to filter on L4");
2379         if (item_flags & l4m)
2380                 return rte_flow_error_set(error, EINVAL,
2381                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2382                                           "multiple L4 layers not supported");
2383         if (!mask)
2384                 mask = &rte_flow_item_tcp_mask;
2385         ret = mlx5_flow_item_acceptable
2386                 (item, (const uint8_t *)mask,
2387                  (const uint8_t *)flow_mask,
2388                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2389                  error);
2390         if (ret < 0)
2391                 return ret;
2392         return 0;
2393 }
2394
2395 /**
2396  * Validate VXLAN item.
2397  *
2398  * @param[in] item
2399  *   Item specification.
2400  * @param[in] item_flags
2401  *   Bit-fields that holds the items detected until now.
2402  * @param[in] target_protocol
2403  *   The next protocol in the previous item.
2404  * @param[out] error
2405  *   Pointer to error structure.
2406  *
2407  * @return
2408  *   0 on success, a negative errno value otherwise and rte_errno is set.
2409  */
2410 int
2411 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
2412                               uint64_t item_flags,
2413                               struct rte_flow_error *error)
2414 {
2415         const struct rte_flow_item_vxlan *spec = item->spec;
2416         const struct rte_flow_item_vxlan *mask = item->mask;
2417         int ret;
2418         union vni {
2419                 uint32_t vlan_id;
2420                 uint8_t vni[4];
2421         } id = { .vlan_id = 0, };
2422
2423
2424         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2425                 return rte_flow_error_set(error, ENOTSUP,
2426                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2427                                           "multiple tunnel layers not"
2428                                           " supported");
2429         /*
2430          * Verify only UDPv4 is present as defined in
2431          * https://tools.ietf.org/html/rfc7348
2432          */
2433         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2434                 return rte_flow_error_set(error, EINVAL,
2435                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2436                                           "no outer UDP layer found");
2437         if (!mask)
2438                 mask = &rte_flow_item_vxlan_mask;
2439         ret = mlx5_flow_item_acceptable
2440                 (item, (const uint8_t *)mask,
2441                  (const uint8_t *)&rte_flow_item_vxlan_mask,
2442                  sizeof(struct rte_flow_item_vxlan),
2443                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2444         if (ret < 0)
2445                 return ret;
2446         if (spec) {
2447                 memcpy(&id.vni[1], spec->vni, 3);
2448                 memcpy(&id.vni[1], mask->vni, 3);
2449         }
2450         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2451                 return rte_flow_error_set(error, ENOTSUP,
2452                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2453                                           "VXLAN tunnel must be fully defined");
2454         return 0;
2455 }
2456
2457 /**
2458  * Validate VXLAN_GPE item.
2459  *
2460  * @param[in] item
2461  *   Item specification.
2462  * @param[in] item_flags
2463  *   Bit-fields that holds the items detected until now.
2464  * @param[in] priv
2465  *   Pointer to the private data structure.
2466  * @param[in] target_protocol
2467  *   The next protocol in the previous item.
2468  * @param[out] error
2469  *   Pointer to error structure.
2470  *
2471  * @return
2472  *   0 on success, a negative errno value otherwise and rte_errno is set.
2473  */
2474 int
2475 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2476                                   uint64_t item_flags,
2477                                   struct rte_eth_dev *dev,
2478                                   struct rte_flow_error *error)
2479 {
2480         struct mlx5_priv *priv = dev->data->dev_private;
2481         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2482         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2483         int ret;
2484         union vni {
2485                 uint32_t vlan_id;
2486                 uint8_t vni[4];
2487         } id = { .vlan_id = 0, };
2488
2489         if (!priv->config.l3_vxlan_en)
2490                 return rte_flow_error_set(error, ENOTSUP,
2491                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2492                                           "L3 VXLAN is not enabled by device"
2493                                           " parameter and/or not configured in"
2494                                           " firmware");
2495         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2496                 return rte_flow_error_set(error, ENOTSUP,
2497                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2498                                           "multiple tunnel layers not"
2499                                           " supported");
2500         /*
2501          * Verify only UDPv4 is present as defined in
2502          * https://tools.ietf.org/html/rfc7348
2503          */
2504         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2505                 return rte_flow_error_set(error, EINVAL,
2506                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2507                                           "no outer UDP layer found");
2508         if (!mask)
2509                 mask = &rte_flow_item_vxlan_gpe_mask;
2510         ret = mlx5_flow_item_acceptable
2511                 (item, (const uint8_t *)mask,
2512                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2513                  sizeof(struct rte_flow_item_vxlan_gpe),
2514                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2515         if (ret < 0)
2516                 return ret;
2517         if (spec) {
2518                 if (spec->protocol)
2519                         return rte_flow_error_set(error, ENOTSUP,
2520                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2521                                                   item,
2522                                                   "VxLAN-GPE protocol"
2523                                                   " not supported");
2524                 memcpy(&id.vni[1], spec->vni, 3);
2525                 memcpy(&id.vni[1], mask->vni, 3);
2526         }
2527         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2528                 return rte_flow_error_set(error, ENOTSUP,
2529                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2530                                           "VXLAN-GPE tunnel must be fully"
2531                                           " defined");
2532         return 0;
2533 }
2534 /**
2535  * Validate GRE Key item.
2536  *
2537  * @param[in] item
2538  *   Item specification.
2539  * @param[in] item_flags
2540  *   Bit flags to mark detected items.
2541  * @param[in] gre_item
2542  *   Pointer to gre_item
2543  * @param[out] error
2544  *   Pointer to error structure.
2545  *
2546  * @return
2547  *   0 on success, a negative errno value otherwise and rte_errno is set.
2548  */
2549 int
2550 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2551                                 uint64_t item_flags,
2552                                 const struct rte_flow_item *gre_item,
2553                                 struct rte_flow_error *error)
2554 {
2555         const rte_be32_t *mask = item->mask;
2556         int ret = 0;
2557         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2558         const struct rte_flow_item_gre *gre_spec;
2559         const struct rte_flow_item_gre *gre_mask;
2560
2561         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2562                 return rte_flow_error_set(error, ENOTSUP,
2563                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2564                                           "Multiple GRE key not support");
2565         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2566                 return rte_flow_error_set(error, ENOTSUP,
2567                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2568                                           "No preceding GRE header");
2569         if (item_flags & MLX5_FLOW_LAYER_INNER)
2570                 return rte_flow_error_set(error, ENOTSUP,
2571                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2572                                           "GRE key following a wrong item");
2573         gre_mask = gre_item->mask;
2574         if (!gre_mask)
2575                 gre_mask = &rte_flow_item_gre_mask;
2576         gre_spec = gre_item->spec;
2577         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2578                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2579                 return rte_flow_error_set(error, EINVAL,
2580                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2581                                           "Key bit must be on");
2582
2583         if (!mask)
2584                 mask = &gre_key_default_mask;
2585         ret = mlx5_flow_item_acceptable
2586                 (item, (const uint8_t *)mask,
2587                  (const uint8_t *)&gre_key_default_mask,
2588                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2589         return ret;
2590 }
2591
2592 /**
2593  * Validate GRE item.
2594  *
2595  * @param[in] item
2596  *   Item specification.
2597  * @param[in] item_flags
2598  *   Bit flags to mark detected items.
2599  * @param[in] target_protocol
2600  *   The next protocol in the previous item.
2601  * @param[out] error
2602  *   Pointer to error structure.
2603  *
2604  * @return
2605  *   0 on success, a negative errno value otherwise and rte_errno is set.
2606  */
2607 int
2608 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2609                             uint64_t item_flags,
2610                             uint8_t target_protocol,
2611                             struct rte_flow_error *error)
2612 {
2613         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2614         const struct rte_flow_item_gre *mask = item->mask;
2615         int ret;
2616         const struct rte_flow_item_gre nic_mask = {
2617                 .c_rsvd0_ver = RTE_BE16(0xB000),
2618                 .protocol = RTE_BE16(UINT16_MAX),
2619         };
2620
2621         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2622                 return rte_flow_error_set(error, EINVAL,
2623                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2624                                           "protocol filtering not compatible"
2625                                           " with this GRE layer");
2626         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2627                 return rte_flow_error_set(error, ENOTSUP,
2628                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2629                                           "multiple tunnel layers not"
2630                                           " supported");
2631         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2632                 return rte_flow_error_set(error, ENOTSUP,
2633                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2634                                           "L3 Layer is missing");
2635         if (!mask)
2636                 mask = &rte_flow_item_gre_mask;
2637         ret = mlx5_flow_item_acceptable
2638                 (item, (const uint8_t *)mask,
2639                  (const uint8_t *)&nic_mask,
2640                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2641                  error);
2642         if (ret < 0)
2643                 return ret;
2644 #ifndef HAVE_MLX5DV_DR
2645 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2646         if (spec && (spec->protocol & mask->protocol))
2647                 return rte_flow_error_set(error, ENOTSUP,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2649                                           "without MPLS support the"
2650                                           " specification cannot be used for"
2651                                           " filtering");
2652 #endif
2653 #endif
2654         return 0;
2655 }
2656
2657 /**
2658  * Validate Geneve item.
2659  *
2660  * @param[in] item
2661  *   Item specification.
2662  * @param[in] itemFlags
2663  *   Bit-fields that holds the items detected until now.
2664  * @param[in] enPriv
2665  *   Pointer to the private data structure.
2666  * @param[out] error
2667  *   Pointer to error structure.
2668  *
2669  * @return
2670  *   0 on success, a negative errno value otherwise and rte_errno is set.
2671  */
2672
2673 int
2674 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2675                                uint64_t item_flags,
2676                                struct rte_eth_dev *dev,
2677                                struct rte_flow_error *error)
2678 {
2679         struct mlx5_priv *priv = dev->data->dev_private;
2680         const struct rte_flow_item_geneve *spec = item->spec;
2681         const struct rte_flow_item_geneve *mask = item->mask;
2682         int ret;
2683         uint16_t gbhdr;
2684         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2685                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2686         const struct rte_flow_item_geneve nic_mask = {
2687                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2688                 .vni = "\xff\xff\xff",
2689                 .protocol = RTE_BE16(UINT16_MAX),
2690         };
2691
2692         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2693                 return rte_flow_error_set(error, ENOTSUP,
2694                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2695                                           "L3 Geneve is not enabled by device"
2696                                           " parameter and/or not configured in"
2697                                           " firmware");
2698         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2699                 return rte_flow_error_set(error, ENOTSUP,
2700                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2701                                           "multiple tunnel layers not"
2702                                           " supported");
2703         /*
2704          * Verify only UDPv4 is present as defined in
2705          * https://tools.ietf.org/html/rfc7348
2706          */
2707         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2708                 return rte_flow_error_set(error, EINVAL,
2709                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2710                                           "no outer UDP layer found");
2711         if (!mask)
2712                 mask = &rte_flow_item_geneve_mask;
2713         ret = mlx5_flow_item_acceptable
2714                                   (item, (const uint8_t *)mask,
2715                                    (const uint8_t *)&nic_mask,
2716                                    sizeof(struct rte_flow_item_geneve),
2717                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2718         if (ret)
2719                 return ret;
2720         if (spec) {
2721                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2722                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2723                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2724                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2725                         return rte_flow_error_set(error, ENOTSUP,
2726                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2727                                                   item,
2728                                                   "Geneve protocol unsupported"
2729                                                   " fields are being used");
2730                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2731                         return rte_flow_error_set
2732                                         (error, ENOTSUP,
2733                                          RTE_FLOW_ERROR_TYPE_ITEM,
2734                                          item,
2735                                          "Unsupported Geneve options length");
2736         }
2737         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2738                 return rte_flow_error_set
2739                                     (error, ENOTSUP,
2740                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2741                                      "Geneve tunnel must be fully defined");
2742         return 0;
2743 }
2744
2745 /**
2746  * Validate Geneve TLV option item.
2747  *
2748  * @param[in] item
2749  *   Item specification.
2750  * @param[in] last_item
2751  *   Previous validated item in the pattern items.
2752  * @param[in] geneve_item
2753  *   Previous GENEVE item specification.
2754  * @param[in] dev
2755  *   Pointer to the rte_eth_dev structure.
2756  * @param[out] error
2757  *   Pointer to error structure.
2758  *
2759  * @return
2760  *   0 on success, a negative errno value otherwise and rte_errno is set.
2761  */
2762 int
2763 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2764                                    uint64_t last_item,
2765                                    const struct rte_flow_item *geneve_item,
2766                                    struct rte_eth_dev *dev,
2767                                    struct rte_flow_error *error)
2768 {
2769         struct mlx5_priv *priv = dev->data->dev_private;
2770         struct mlx5_dev_ctx_shared *sh = priv->sh;
2771         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2772         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2773         uint8_t data_max_supported =
2774                         hca_attr->max_geneve_tlv_option_data_len * 4;
2775         struct mlx5_dev_config *config = &priv->config;
2776         const struct rte_flow_item_geneve *geneve_spec;
2777         const struct rte_flow_item_geneve *geneve_mask;
2778         const struct rte_flow_item_geneve_opt *spec = item->spec;
2779         const struct rte_flow_item_geneve_opt *mask = item->mask;
2780         unsigned int i;
2781         unsigned int data_len;
2782         uint8_t tlv_option_len;
2783         uint16_t optlen_m, optlen_v;
2784         const struct rte_flow_item_geneve_opt full_mask = {
2785                 .option_class = RTE_BE16(0xffff),
2786                 .option_type = 0xff,
2787                 .option_len = 0x1f,
2788         };
2789
2790         if (!mask)
2791                 mask = &rte_flow_item_geneve_opt_mask;
2792         if (!spec)
2793                 return rte_flow_error_set
2794                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2795                         "Geneve TLV opt class/type/length must be specified");
2796         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2797                 return rte_flow_error_set
2798                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2799                         "Geneve TLV opt length exceeeds the limit (31)");
2800         /* Check if class type and length masks are full. */
2801         if (full_mask.option_class != mask->option_class ||
2802             full_mask.option_type != mask->option_type ||
2803             full_mask.option_len != (mask->option_len & full_mask.option_len))
2804                 return rte_flow_error_set
2805                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2806                         "Geneve TLV opt class/type/length masks must be full");
2807         /* Check if length is supported */
2808         if ((uint32_t)spec->option_len >
2809                         config->hca_attr.max_geneve_tlv_option_data_len)
2810                 return rte_flow_error_set
2811                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2812                         "Geneve TLV opt length not supported");
2813         if (config->hca_attr.max_geneve_tlv_options > 1)
2814                 DRV_LOG(DEBUG,
2815                         "max_geneve_tlv_options supports more than 1 option");
2816         /* Check GENEVE item preceding. */
2817         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2818                 return rte_flow_error_set
2819                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2820                         "Geneve opt item must be preceded with Geneve item");
2821         geneve_spec = geneve_item->spec;
2822         geneve_mask = geneve_item->mask ? geneve_item->mask :
2823                                           &rte_flow_item_geneve_mask;
2824         /* Check if GENEVE TLV option size doesn't exceed option length */
2825         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2826                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2827                 tlv_option_len = spec->option_len & mask->option_len;
2828                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2829                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2830                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2831                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2832                 if ((optlen_v & optlen_m) <= tlv_option_len)
2833                         return rte_flow_error_set
2834                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2835                                  "GENEVE TLV option length exceeds optlen");
2836         }
2837         /* Check if length is 0 or data is 0. */
2838         if (spec->data == NULL || spec->option_len == 0)
2839                 return rte_flow_error_set
2840                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2841                         "Geneve TLV opt with zero data/length not supported");
2842         /* Check not all data & mask are 0. */
2843         data_len = spec->option_len * 4;
2844         if (mask->data == NULL) {
2845                 for (i = 0; i < data_len; i++)
2846                         if (spec->data[i])
2847                                 break;
2848                 if (i == data_len)
2849                         return rte_flow_error_set(error, ENOTSUP,
2850                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2851                                 "Can't match on Geneve option data 0");
2852         } else {
2853                 for (i = 0; i < data_len; i++)
2854                         if (spec->data[i] & mask->data[i])
2855                                 break;
2856                 if (i == data_len)
2857                         return rte_flow_error_set(error, ENOTSUP,
2858                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2859                                 "Can't match on Geneve option data and mask 0");
2860                 /* Check data mask supported. */
2861                 for (i = data_max_supported; i < data_len ; i++)
2862                         if (mask->data[i])
2863                                 return rte_flow_error_set(error, ENOTSUP,
2864                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2865                                         "Data mask is of unsupported size");
2866         }
2867         /* Check GENEVE option is supported in NIC. */
2868         if (!config->hca_attr.geneve_tlv_opt)
2869                 return rte_flow_error_set
2870                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2871                         "Geneve TLV opt not supported");
2872         /* Check if we already have geneve option with different type/class. */
2873         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2874         geneve_opt_resource = sh->geneve_tlv_option_resource;
2875         if (geneve_opt_resource != NULL)
2876                 if (geneve_opt_resource->option_class != spec->option_class ||
2877                     geneve_opt_resource->option_type != spec->option_type ||
2878                     geneve_opt_resource->length != spec->option_len) {
2879                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2880                         return rte_flow_error_set(error, ENOTSUP,
2881                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2882                                 "Only one Geneve TLV option supported");
2883                 }
2884         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2885         return 0;
2886 }
2887
2888 /**
2889  * Validate MPLS item.
2890  *
2891  * @param[in] dev
2892  *   Pointer to the rte_eth_dev structure.
2893  * @param[in] item
2894  *   Item specification.
2895  * @param[in] item_flags
2896  *   Bit-fields that holds the items detected until now.
2897  * @param[in] prev_layer
2898  *   The protocol layer indicated in previous item.
2899  * @param[out] error
2900  *   Pointer to error structure.
2901  *
2902  * @return
2903  *   0 on success, a negative errno value otherwise and rte_errno is set.
2904  */
2905 int
2906 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2907                              const struct rte_flow_item *item __rte_unused,
2908                              uint64_t item_flags __rte_unused,
2909                              uint64_t prev_layer __rte_unused,
2910                              struct rte_flow_error *error)
2911 {
2912 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2913         const struct rte_flow_item_mpls *mask = item->mask;
2914         struct mlx5_priv *priv = dev->data->dev_private;
2915         int ret;
2916
2917         if (!priv->config.mpls_en)
2918                 return rte_flow_error_set(error, ENOTSUP,
2919                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2920                                           "MPLS not supported or"
2921                                           " disabled in firmware"
2922                                           " configuration.");
2923         /* MPLS over IP, UDP, GRE is allowed */
2924         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2925                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2926                             MLX5_FLOW_LAYER_GRE |
2927                             MLX5_FLOW_LAYER_GRE_KEY)))
2928                 return rte_flow_error_set(error, EINVAL,
2929                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2930                                           "protocol filtering not compatible"
2931                                           " with MPLS layer");
2932         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2933         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2934             !(item_flags & MLX5_FLOW_LAYER_GRE))
2935                 return rte_flow_error_set(error, ENOTSUP,
2936                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2937                                           "multiple tunnel layers not"
2938                                           " supported");
2939         if (!mask)
2940                 mask = &rte_flow_item_mpls_mask;
2941         ret = mlx5_flow_item_acceptable
2942                 (item, (const uint8_t *)mask,
2943                  (const uint8_t *)&rte_flow_item_mpls_mask,
2944                  sizeof(struct rte_flow_item_mpls),
2945                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2946         if (ret < 0)
2947                 return ret;
2948         return 0;
2949 #else
2950         return rte_flow_error_set(error, ENOTSUP,
2951                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2952                                   "MPLS is not supported by Verbs, please"
2953                                   " update.");
2954 #endif
2955 }
2956
2957 /**
2958  * Validate NVGRE item.
2959  *
2960  * @param[in] item
2961  *   Item specification.
2962  * @param[in] item_flags
2963  *   Bit flags to mark detected items.
2964  * @param[in] target_protocol
2965  *   The next protocol in the previous item.
2966  * @param[out] error
2967  *   Pointer to error structure.
2968  *
2969  * @return
2970  *   0 on success, a negative errno value otherwise and rte_errno is set.
2971  */
2972 int
2973 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2974                               uint64_t item_flags,
2975                               uint8_t target_protocol,
2976                               struct rte_flow_error *error)
2977 {
2978         const struct rte_flow_item_nvgre *mask = item->mask;
2979         int ret;
2980
2981         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2982                 return rte_flow_error_set(error, EINVAL,
2983                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2984                                           "protocol filtering not compatible"
2985                                           " with this GRE layer");
2986         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2987                 return rte_flow_error_set(error, ENOTSUP,
2988                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2989                                           "multiple tunnel layers not"
2990                                           " supported");
2991         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2992                 return rte_flow_error_set(error, ENOTSUP,
2993                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2994                                           "L3 Layer is missing");
2995         if (!mask)
2996                 mask = &rte_flow_item_nvgre_mask;
2997         ret = mlx5_flow_item_acceptable
2998                 (item, (const uint8_t *)mask,
2999                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3000                  sizeof(struct rte_flow_item_nvgre),
3001                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3002         if (ret < 0)
3003                 return ret;
3004         return 0;
3005 }
3006
3007 /**
3008  * Validate eCPRI item.
3009  *
3010  * @param[in] item
3011  *   Item specification.
3012  * @param[in] item_flags
3013  *   Bit-fields that holds the items detected until now.
3014  * @param[in] last_item
3015  *   Previous validated item in the pattern items.
3016  * @param[in] ether_type
3017  *   Type in the ethernet layer header (including dot1q).
3018  * @param[in] acc_mask
3019  *   Acceptable mask, if NULL default internal default mask
3020  *   will be used to check whether item fields are supported.
3021  * @param[out] error
3022  *   Pointer to error structure.
3023  *
3024  * @return
3025  *   0 on success, a negative errno value otherwise and rte_errno is set.
3026  */
3027 int
3028 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3029                               uint64_t item_flags,
3030                               uint64_t last_item,
3031                               uint16_t ether_type,
3032                               const struct rte_flow_item_ecpri *acc_mask,
3033                               struct rte_flow_error *error)
3034 {
3035         const struct rte_flow_item_ecpri *mask = item->mask;
3036         const struct rte_flow_item_ecpri nic_mask = {
3037                 .hdr = {
3038                         .common = {
3039                                 .u32 =
3040                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3041                                         .type = 0xFF,
3042                                         }).u32),
3043                         },
3044                         .dummy[0] = 0xFFFFFFFF,
3045                 },
3046         };
3047         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3048                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3049         struct rte_flow_item_ecpri mask_lo;
3050
3051         if (!(last_item & outer_l2_vlan) &&
3052             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3053                 return rte_flow_error_set(error, EINVAL,
3054                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3055                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3056         if ((last_item & outer_l2_vlan) && ether_type &&
3057             ether_type != RTE_ETHER_TYPE_ECPRI)
3058                 return rte_flow_error_set(error, EINVAL,
3059                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3060                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3061         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3062                 return rte_flow_error_set(error, EINVAL,
3063                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3064                                           "eCPRI with tunnel is not supported right now");
3065         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3066                 return rte_flow_error_set(error, ENOTSUP,
3067                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3068                                           "multiple L3 layers not supported");
3069         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3070                 return rte_flow_error_set(error, EINVAL,
3071                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3072                                           "eCPRI cannot coexist with a TCP layer");
3073         /* In specification, eCPRI could be over UDP layer. */
3074         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3075                 return rte_flow_error_set(error, EINVAL,
3076                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3077                                           "eCPRI over UDP layer is not yet supported right now");
3078         /* Mask for type field in common header could be zero. */
3079         if (!mask)
3080                 mask = &rte_flow_item_ecpri_mask;
3081         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3082         /* Input mask is in big-endian format. */
3083         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3084                 return rte_flow_error_set(error, EINVAL,
3085                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3086                                           "partial mask is not supported for protocol");
3087         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3088                 return rte_flow_error_set(error, EINVAL,
3089                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3090                                           "message header mask must be after a type mask");
3091         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3092                                          acc_mask ? (const uint8_t *)acc_mask
3093                                                   : (const uint8_t *)&nic_mask,
3094                                          sizeof(struct rte_flow_item_ecpri),
3095                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3096 }
3097
3098 /**
3099  * Release resource related QUEUE/RSS action split.
3100  *
3101  * @param dev
3102  *   Pointer to Ethernet device.
3103  * @param flow
3104  *   Flow to release id's from.
3105  */
3106 static void
3107 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
3108                              struct rte_flow *flow)
3109 {
3110         struct mlx5_priv *priv = dev->data->dev_private;
3111         uint32_t handle_idx;
3112         struct mlx5_flow_handle *dev_handle;
3113
3114         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
3115                        handle_idx, dev_handle, next)
3116                 if (dev_handle->split_flow_id &&
3117                     !dev_handle->is_meter_flow_id)
3118                         mlx5_ipool_free(priv->sh->ipool
3119                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
3120                                         dev_handle->split_flow_id);
3121 }
3122
3123 static int
3124 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3125                    const struct rte_flow_attr *attr __rte_unused,
3126                    const struct rte_flow_item items[] __rte_unused,
3127                    const struct rte_flow_action actions[] __rte_unused,
3128                    bool external __rte_unused,
3129                    int hairpin __rte_unused,
3130                    struct rte_flow_error *error)
3131 {
3132         return rte_flow_error_set(error, ENOTSUP,
3133                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3134 }
3135
3136 static struct mlx5_flow *
3137 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3138                   const struct rte_flow_attr *attr __rte_unused,
3139                   const struct rte_flow_item items[] __rte_unused,
3140                   const struct rte_flow_action actions[] __rte_unused,
3141                   struct rte_flow_error *error)
3142 {
3143         rte_flow_error_set(error, ENOTSUP,
3144                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3145         return NULL;
3146 }
3147
3148 static int
3149 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3150                     struct mlx5_flow *dev_flow __rte_unused,
3151                     const struct rte_flow_attr *attr __rte_unused,
3152                     const struct rte_flow_item items[] __rte_unused,
3153                     const struct rte_flow_action actions[] __rte_unused,
3154                     struct rte_flow_error *error)
3155 {
3156         return rte_flow_error_set(error, ENOTSUP,
3157                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3158 }
3159
3160 static int
3161 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3162                 struct rte_flow *flow __rte_unused,
3163                 struct rte_flow_error *error)
3164 {
3165         return rte_flow_error_set(error, ENOTSUP,
3166                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3167 }
3168
3169 static void
3170 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3171                  struct rte_flow *flow __rte_unused)
3172 {
3173 }
3174
3175 static void
3176 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3177                   struct rte_flow *flow __rte_unused)
3178 {
3179 }
3180
3181 static int
3182 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3183                 struct rte_flow *flow __rte_unused,
3184                 const struct rte_flow_action *actions __rte_unused,
3185                 void *data __rte_unused,
3186                 struct rte_flow_error *error)
3187 {
3188         return rte_flow_error_set(error, ENOTSUP,
3189                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3190 }
3191
3192 static int
3193 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3194                       uint32_t domains __rte_unused,
3195                       uint32_t flags __rte_unused)
3196 {
3197         return 0;
3198 }
3199
3200 /* Void driver to protect from null pointer reference. */
3201 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3202         .validate = flow_null_validate,
3203         .prepare = flow_null_prepare,
3204         .translate = flow_null_translate,
3205         .apply = flow_null_apply,
3206         .remove = flow_null_remove,
3207         .destroy = flow_null_destroy,
3208         .query = flow_null_query,
3209         .sync_domain = flow_null_sync_domain,
3210 };
3211
3212 /**
3213  * Select flow driver type according to flow attributes and device
3214  * configuration.
3215  *
3216  * @param[in] dev
3217  *   Pointer to the dev structure.
3218  * @param[in] attr
3219  *   Pointer to the flow attributes.
3220  *
3221  * @return
3222  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3223  */
3224 static enum mlx5_flow_drv_type
3225 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3226 {
3227         struct mlx5_priv *priv = dev->data->dev_private;
3228         /* The OS can determine first a specific flow type (DV, VERBS) */
3229         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3230
3231         if (type != MLX5_FLOW_TYPE_MAX)
3232                 return type;
3233         /* If no OS specific type - continue with DV/VERBS selection */
3234         if (attr->transfer && priv->config.dv_esw_en)
3235                 type = MLX5_FLOW_TYPE_DV;
3236         if (!attr->transfer)
3237                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3238                                                  MLX5_FLOW_TYPE_VERBS;
3239         return type;
3240 }
3241
3242 #define flow_get_drv_ops(type) flow_drv_ops[type]
3243
3244 /**
3245  * Flow driver validation API. This abstracts calling driver specific functions.
3246  * The type of flow driver is determined according to flow attributes.
3247  *
3248  * @param[in] dev
3249  *   Pointer to the dev structure.
3250  * @param[in] attr
3251  *   Pointer to the flow attributes.
3252  * @param[in] items
3253  *   Pointer to the list of items.
3254  * @param[in] actions
3255  *   Pointer to the list of actions.
3256  * @param[in] external
3257  *   This flow rule is created by request external to PMD.
3258  * @param[in] hairpin
3259  *   Number of hairpin TX actions, 0 means classic flow.
3260  * @param[out] error
3261  *   Pointer to the error structure.
3262  *
3263  * @return
3264  *   0 on success, a negative errno value otherwise and rte_errno is set.
3265  */
3266 static inline int
3267 flow_drv_validate(struct rte_eth_dev *dev,
3268                   const struct rte_flow_attr *attr,
3269                   const struct rte_flow_item items[],
3270                   const struct rte_flow_action actions[],
3271                   bool external, int hairpin, struct rte_flow_error *error)
3272 {
3273         const struct mlx5_flow_driver_ops *fops;
3274         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3275
3276         fops = flow_get_drv_ops(type);
3277         return fops->validate(dev, attr, items, actions, external,
3278                               hairpin, error);
3279 }
3280
3281 /**
3282  * Flow driver preparation API. This abstracts calling driver specific
3283  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3284  * calculates the size of memory required for device flow, allocates the memory,
3285  * initializes the device flow and returns the pointer.
3286  *
3287  * @note
3288  *   This function initializes device flow structure such as dv or verbs in
3289  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3290  *   rest. For example, adding returning device flow to flow->dev_flow list and
3291  *   setting backward reference to the flow should be done out of this function.
3292  *   layers field is not filled either.
3293  *
3294  * @param[in] dev
3295  *   Pointer to the dev structure.
3296  * @param[in] attr
3297  *   Pointer to the flow attributes.
3298  * @param[in] items
3299  *   Pointer to the list of items.
3300  * @param[in] actions
3301  *   Pointer to the list of actions.
3302  * @param[in] flow_idx
3303  *   This memory pool index to the flow.
3304  * @param[out] error
3305  *   Pointer to the error structure.
3306  *
3307  * @return
3308  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3309  */
3310 static inline struct mlx5_flow *
3311 flow_drv_prepare(struct rte_eth_dev *dev,
3312                  const struct rte_flow *flow,
3313                  const struct rte_flow_attr *attr,
3314                  const struct rte_flow_item items[],
3315                  const struct rte_flow_action actions[],
3316                  uint32_t flow_idx,
3317                  struct rte_flow_error *error)
3318 {
3319         const struct mlx5_flow_driver_ops *fops;
3320         enum mlx5_flow_drv_type type = flow->drv_type;
3321         struct mlx5_flow *mlx5_flow = NULL;
3322
3323         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3324         fops = flow_get_drv_ops(type);
3325         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3326         if (mlx5_flow)
3327                 mlx5_flow->flow_idx = flow_idx;
3328         return mlx5_flow;
3329 }
3330
3331 /**
3332  * Flow driver translation API. This abstracts calling driver specific
3333  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3334  * translates a generic flow into a driver flow. flow_drv_prepare() must
3335  * precede.
3336  *
3337  * @note
3338  *   dev_flow->layers could be filled as a result of parsing during translation
3339  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3340  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3341  *   flow->actions could be overwritten even though all the expanded dev_flows
3342  *   have the same actions.
3343  *
3344  * @param[in] dev
3345  *   Pointer to the rte dev structure.
3346  * @param[in, out] dev_flow
3347  *   Pointer to the mlx5 flow.
3348  * @param[in] attr
3349  *   Pointer to the flow attributes.
3350  * @param[in] items
3351  *   Pointer to the list of items.
3352  * @param[in] actions
3353  *   Pointer to the list of actions.
3354  * @param[out] error
3355  *   Pointer to the error structure.
3356  *
3357  * @return
3358  *   0 on success, a negative errno value otherwise and rte_errno is set.
3359  */
3360 static inline int
3361 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3362                    const struct rte_flow_attr *attr,
3363                    const struct rte_flow_item items[],
3364                    const struct rte_flow_action actions[],
3365                    struct rte_flow_error *error)
3366 {
3367         const struct mlx5_flow_driver_ops *fops;
3368         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3369
3370         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3371         fops = flow_get_drv_ops(type);
3372         return fops->translate(dev, dev_flow, attr, items, actions, error);
3373 }
3374
3375 /**
3376  * Flow driver apply API. This abstracts calling driver specific functions.
3377  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3378  * translated driver flows on to device. flow_drv_translate() must precede.
3379  *
3380  * @param[in] dev
3381  *   Pointer to Ethernet device structure.
3382  * @param[in, out] flow
3383  *   Pointer to flow structure.
3384  * @param[out] error
3385  *   Pointer to error structure.
3386  *
3387  * @return
3388  *   0 on success, a negative errno value otherwise and rte_errno is set.
3389  */
3390 static inline int
3391 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3392                struct rte_flow_error *error)
3393 {
3394         const struct mlx5_flow_driver_ops *fops;
3395         enum mlx5_flow_drv_type type = flow->drv_type;
3396
3397         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3398         fops = flow_get_drv_ops(type);
3399         return fops->apply(dev, flow, error);
3400 }
3401
3402 /**
3403  * Flow driver destroy API. This abstracts calling driver specific functions.
3404  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3405  * on device and releases resources of the flow.
3406  *
3407  * @param[in] dev
3408  *   Pointer to Ethernet device.
3409  * @param[in, out] flow
3410  *   Pointer to flow structure.
3411  */
3412 static inline void
3413 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3414 {
3415         const struct mlx5_flow_driver_ops *fops;
3416         enum mlx5_flow_drv_type type = flow->drv_type;
3417
3418         flow_mreg_split_qrss_release(dev, flow);
3419         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3420         fops = flow_get_drv_ops(type);
3421         fops->destroy(dev, flow);
3422 }
3423
3424 /**
3425  * Flow driver find RSS policy tbl API. This abstracts calling driver
3426  * specific functions. Parent flow (rte_flow) should have driver
3427  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3428  *
3429  * @param[in] dev
3430  *   Pointer to Ethernet device.
3431  * @param[in, out] flow
3432  *   Pointer to flow structure.
3433  * @param[in] policy
3434  *   Pointer to meter policy table.
3435  * @param[in] rss_desc
3436  *   Pointer to rss_desc
3437  */
3438 static struct mlx5_flow_meter_sub_policy *
3439 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3440                 struct rte_flow *flow,
3441                 struct mlx5_flow_meter_policy *policy,
3442                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3443 {
3444         const struct mlx5_flow_driver_ops *fops;
3445         enum mlx5_flow_drv_type type = flow->drv_type;
3446
3447         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3448         fops = flow_get_drv_ops(type);
3449         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3450 }
3451
3452 /**
3453  * Get RSS action from the action list.
3454  *
3455  * @param[in] dev
3456  *   Pointer to Ethernet device.
3457  * @param[in] actions
3458  *   Pointer to the list of actions.
3459  * @param[in] flow
3460  *   Parent flow structure pointer.
3461  *
3462  * @return
3463  *   Pointer to the RSS action if exist, else return NULL.
3464  */
3465 static const struct rte_flow_action_rss*
3466 flow_get_rss_action(struct rte_eth_dev *dev,
3467                     const struct rte_flow_action actions[])
3468 {
3469         struct mlx5_priv *priv = dev->data->dev_private;
3470         const struct rte_flow_action_rss *rss = NULL;
3471
3472         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3473                 switch (actions->type) {
3474                 case RTE_FLOW_ACTION_TYPE_RSS:
3475                         rss = actions->conf;
3476                         break;
3477                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3478                 {
3479                         const struct rte_flow_action_sample *sample =
3480                                                                 actions->conf;
3481                         const struct rte_flow_action *act = sample->actions;
3482                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3483                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3484                                         rss = act->conf;
3485                         break;
3486                 }
3487                 case RTE_FLOW_ACTION_TYPE_METER:
3488                 {
3489                         uint32_t mtr_idx;
3490                         struct mlx5_flow_meter_info *fm;
3491                         struct mlx5_flow_meter_policy *policy;
3492                         const struct rte_flow_action_meter *mtr = actions->conf;
3493
3494                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3495                         if (fm) {
3496                                 policy = mlx5_flow_meter_policy_find(dev,
3497                                                 fm->policy_id, NULL);
3498                                 if (policy && policy->is_rss)
3499                                         rss =
3500                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3501                         }
3502                         break;
3503                 }
3504                 default:
3505                         break;
3506                 }
3507         }
3508         return rss;
3509 }
3510
3511 /**
3512  * Get ASO age action by index.
3513  *
3514  * @param[in] dev
3515  *   Pointer to the Ethernet device structure.
3516  * @param[in] age_idx
3517  *   Index to the ASO age action.
3518  *
3519  * @return
3520  *   The specified ASO age action.
3521  */
3522 struct mlx5_aso_age_action*
3523 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3524 {
3525         uint16_t pool_idx = age_idx & UINT16_MAX;
3526         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3527         struct mlx5_priv *priv = dev->data->dev_private;
3528         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3529         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3530
3531         return &pool->actions[offset - 1];
3532 }
3533
3534 /* maps indirect action to translated direct in some actions array */
3535 struct mlx5_translated_action_handle {
3536         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3537         int index; /**< Index in related array of rte_flow_action. */
3538 };
3539
3540 /**
3541  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3542  * direct action if translation possible.
3543  * This functionality used to run same execution path for both direct and
3544  * indirect actions on flow create. All necessary preparations for indirect
3545  * action handling should be performed on *handle* actions list returned
3546  * from this call.
3547  *
3548  * @param[in] dev
3549  *   Pointer to Ethernet device.
3550  * @param[in] actions
3551  *   List of actions to translate.
3552  * @param[out] handle
3553  *   List to store translated indirect action object handles.
3554  * @param[in, out] indir_n
3555  *   Size of *handle* array. On return should be updated with number of
3556  *   indirect actions retrieved from the *actions* list.
3557  * @param[out] translated_actions
3558  *   List of actions where all indirect actions were translated to direct
3559  *   if possible. NULL if no translation took place.
3560  * @param[out] error
3561  *   Pointer to the error structure.
3562  *
3563  * @return
3564  *   0 on success, a negative errno value otherwise and rte_errno is set.
3565  */
3566 static int
3567 flow_action_handles_translate(struct rte_eth_dev *dev,
3568                               const struct rte_flow_action actions[],
3569                               struct mlx5_translated_action_handle *handle,
3570                               int *indir_n,
3571                               struct rte_flow_action **translated_actions,
3572                               struct rte_flow_error *error)
3573 {
3574         struct mlx5_priv *priv = dev->data->dev_private;
3575         struct rte_flow_action *translated = NULL;
3576         size_t actions_size;
3577         int n;
3578         int copied_n = 0;
3579         struct mlx5_translated_action_handle *handle_end = NULL;
3580
3581         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3582                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3583                         continue;
3584                 if (copied_n == *indir_n) {
3585                         return rte_flow_error_set
3586                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3587                                  NULL, "too many shared actions");
3588                 }
3589                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3590                            sizeof(actions[n].conf));
3591                 handle[copied_n].index = n;
3592                 copied_n++;
3593         }
3594         n++;
3595         *indir_n = copied_n;
3596         if (!copied_n)
3597                 return 0;
3598         actions_size = sizeof(struct rte_flow_action) * n;
3599         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3600         if (!translated) {
3601                 rte_errno = ENOMEM;
3602                 return -ENOMEM;
3603         }
3604         memcpy(translated, actions, actions_size);
3605         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3606                 struct mlx5_shared_action_rss *shared_rss;
3607                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3608                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3609                 uint32_t idx = act_idx &
3610                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3611
3612                 switch (type) {
3613                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3614                         shared_rss = mlx5_ipool_get
3615                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3616                         translated[handle->index].type =
3617                                 RTE_FLOW_ACTION_TYPE_RSS;
3618                         translated[handle->index].conf =
3619                                 &shared_rss->origin;
3620                         break;
3621                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3622                         translated[handle->index].type =
3623                                                 (enum rte_flow_action_type)
3624                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3625                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3626                         break;
3627                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3628                         if (priv->sh->flow_hit_aso_en) {
3629                                 translated[handle->index].type =
3630                                         (enum rte_flow_action_type)
3631                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3632                                 translated[handle->index].conf =
3633                                                          (void *)(uintptr_t)idx;
3634                                 break;
3635                         }
3636                         /* Fall-through */
3637                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3638                         if (priv->sh->ct_aso_en) {
3639                                 translated[handle->index].type =
3640                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3641                                 translated[handle->index].conf =
3642                                                          (void *)(uintptr_t)idx;
3643                                 break;
3644                         }
3645                         /* Fall-through */
3646                 default:
3647                         mlx5_free(translated);
3648                         return rte_flow_error_set
3649                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3650                                  NULL, "invalid indirect action type");
3651                 }
3652         }
3653         *translated_actions = translated;
3654         return 0;
3655 }
3656
3657 /**
3658  * Get Shared RSS action from the action list.
3659  *
3660  * @param[in] dev
3661  *   Pointer to Ethernet device.
3662  * @param[in] shared
3663  *   Pointer to the list of actions.
3664  * @param[in] shared_n
3665  *   Actions list length.
3666  *
3667  * @return
3668  *   The MLX5 RSS action ID if exists, otherwise return 0.
3669  */
3670 static uint32_t
3671 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3672                            struct mlx5_translated_action_handle *handle,
3673                            int shared_n)
3674 {
3675         struct mlx5_translated_action_handle *handle_end;
3676         struct mlx5_priv *priv = dev->data->dev_private;
3677         struct mlx5_shared_action_rss *shared_rss;
3678
3679
3680         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3681                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3682                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3683                 uint32_t idx = act_idx &
3684                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3685                 switch (type) {
3686                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3687                         shared_rss = mlx5_ipool_get
3688                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3689                                                                            idx);
3690                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3691                                            __ATOMIC_RELAXED);
3692                         return idx;
3693                 default:
3694                         break;
3695                 }
3696         }
3697         return 0;
3698 }
3699
3700 static unsigned int
3701 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
3702 {
3703         const struct rte_flow_item *item;
3704         unsigned int has_vlan = 0;
3705
3706         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
3707                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
3708                         has_vlan = 1;
3709                         break;
3710                 }
3711         }
3712         if (has_vlan)
3713                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
3714                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
3715         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3716                                MLX5_EXPANSION_ROOT_OUTER;
3717 }
3718
3719 /**
3720  *  Get layer flags from the prefix flow.
3721  *
3722  *  Some flows may be split to several subflows, the prefix subflow gets the
3723  *  match items and the suffix sub flow gets the actions.
3724  *  Some actions need the user defined match item flags to get the detail for
3725  *  the action.
3726  *  This function helps the suffix flow to get the item layer flags from prefix
3727  *  subflow.
3728  *
3729  * @param[in] dev_flow
3730  *   Pointer the created preifx subflow.
3731  *
3732  * @return
3733  *   The layers get from prefix subflow.
3734  */
3735 static inline uint64_t
3736 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3737 {
3738         uint64_t layers = 0;
3739
3740         /*
3741          * Layers bits could be localization, but usually the compiler will
3742          * help to do the optimization work for source code.
3743          * If no decap actions, use the layers directly.
3744          */
3745         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3746                 return dev_flow->handle->layers;
3747         /* Convert L3 layers with decap action. */
3748         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3749                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3750         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3751                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3752         /* Convert L4 layers with decap action.  */
3753         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3754                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3755         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3756                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3757         return layers;
3758 }
3759
3760 /**
3761  * Get metadata split action information.
3762  *
3763  * @param[in] actions
3764  *   Pointer to the list of actions.
3765  * @param[out] qrss
3766  *   Pointer to the return pointer.
3767  * @param[out] qrss_type
3768  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3769  *   if no QUEUE/RSS is found.
3770  * @param[out] encap_idx
3771  *   Pointer to the index of the encap action if exists, otherwise the last
3772  *   action index.
3773  *
3774  * @return
3775  *   Total number of actions.
3776  */
3777 static int
3778 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3779                                        const struct rte_flow_action **qrss,
3780                                        int *encap_idx)
3781 {
3782         const struct rte_flow_action_raw_encap *raw_encap;
3783         int actions_n = 0;
3784         int raw_decap_idx = -1;
3785
3786         *encap_idx = -1;
3787         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3788                 switch (actions->type) {
3789                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3790                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3791                         *encap_idx = actions_n;
3792                         break;
3793                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3794                         raw_decap_idx = actions_n;
3795                         break;
3796                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3797                         raw_encap = actions->conf;
3798                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3799                                 *encap_idx = raw_decap_idx != -1 ?
3800                                                       raw_decap_idx : actions_n;
3801                         break;
3802                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3803                 case RTE_FLOW_ACTION_TYPE_RSS:
3804                         *qrss = actions;
3805                         break;
3806                 default:
3807                         break;
3808                 }
3809                 actions_n++;
3810         }
3811         if (*encap_idx == -1)
3812                 *encap_idx = actions_n;
3813         /* Count RTE_FLOW_ACTION_TYPE_END. */
3814         return actions_n + 1;
3815 }
3816
3817 /**
3818  * Check if the action will change packet.
3819  *
3820  * @param dev
3821  *   Pointer to Ethernet device.
3822  * @param[in] type
3823  *   action type.
3824  *
3825  * @return
3826  *   true if action will change packet, false otherwise.
3827  */
3828 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3829                                           enum rte_flow_action_type type)
3830 {
3831         struct mlx5_priv *priv = dev->data->dev_private;
3832
3833         switch (type) {
3834         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3835         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3836         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3837         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3838         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3839         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3840         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3841         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3842         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3843         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3844         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3845         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3846         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3847         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3848         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3849         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3850         case RTE_FLOW_ACTION_TYPE_SET_META:
3851         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3852         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3853         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3854         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3855         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3856         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3857         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3858         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3859         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3860         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3861         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3862         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3863                 return true;
3864         case RTE_FLOW_ACTION_TYPE_FLAG:
3865         case RTE_FLOW_ACTION_TYPE_MARK:
3866                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3867                         return true;
3868                 else
3869                         return false;
3870         default:
3871                 return false;
3872         }
3873 }
3874
3875 /**
3876  * Check meter action from the action list.
3877  *
3878  * @param dev
3879  *   Pointer to Ethernet device.
3880  * @param[in] actions
3881  *   Pointer to the list of actions.
3882  * @param[out] has_mtr
3883  *   Pointer to the meter exist flag.
3884  * @param[out] has_modify
3885  *   Pointer to the flag showing there's packet change action.
3886  * @param[out] meter_id
3887  *   Pointer to the meter id.
3888  *
3889  * @return
3890  *   Total number of actions.
3891  */
3892 static int
3893 flow_check_meter_action(struct rte_eth_dev *dev,
3894                         const struct rte_flow_action actions[],
3895                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3896 {
3897         const struct rte_flow_action_meter *mtr = NULL;
3898         int actions_n = 0;
3899
3900         MLX5_ASSERT(has_mtr);
3901         *has_mtr = false;
3902         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3903                 switch (actions->type) {
3904                 case RTE_FLOW_ACTION_TYPE_METER:
3905                         mtr = actions->conf;
3906                         *meter_id = mtr->mtr_id;
3907                         *has_mtr = true;
3908                         break;
3909                 default:
3910                         break;
3911                 }
3912                 if (!*has_mtr)
3913                         *has_modify |= flow_check_modify_action_type(dev,
3914                                                                 actions->type);
3915                 actions_n++;
3916         }
3917         /* Count RTE_FLOW_ACTION_TYPE_END. */
3918         return actions_n + 1;
3919 }
3920
3921 /**
3922  * Check if the flow should be split due to hairpin.
3923  * The reason for the split is that in current HW we can't
3924  * support encap and push-vlan on Rx, so if a flow contains
3925  * these actions we move it to Tx.
3926  *
3927  * @param dev
3928  *   Pointer to Ethernet device.
3929  * @param[in] attr
3930  *   Flow rule attributes.
3931  * @param[in] actions
3932  *   Associated actions (list terminated by the END action).
3933  *
3934  * @return
3935  *   > 0 the number of actions and the flow should be split,
3936  *   0 when no split required.
3937  */
3938 static int
3939 flow_check_hairpin_split(struct rte_eth_dev *dev,
3940                          const struct rte_flow_attr *attr,
3941                          const struct rte_flow_action actions[])
3942 {
3943         int queue_action = 0;
3944         int action_n = 0;
3945         int split = 0;
3946         const struct rte_flow_action_queue *queue;
3947         const struct rte_flow_action_rss *rss;
3948         const struct rte_flow_action_raw_encap *raw_encap;
3949         const struct rte_eth_hairpin_conf *conf;
3950
3951         if (!attr->ingress)
3952                 return 0;
3953         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3954                 switch (actions->type) {
3955                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3956                         queue = actions->conf;
3957                         if (queue == NULL)
3958                                 return 0;
3959                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
3960                         if (conf == NULL || conf->tx_explicit != 0)
3961                                 return 0;
3962                         queue_action = 1;
3963                         action_n++;
3964                         break;
3965                 case RTE_FLOW_ACTION_TYPE_RSS:
3966                         rss = actions->conf;
3967                         if (rss == NULL || rss->queue_num == 0)
3968                                 return 0;
3969                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
3970                         if (conf == NULL || conf->tx_explicit != 0)
3971                                 return 0;
3972                         queue_action = 1;
3973                         action_n++;
3974                         break;
3975                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3976                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3977                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3978                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3979                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3980                         split++;
3981                         action_n++;
3982                         break;
3983                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3984                         raw_encap = actions->conf;
3985                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3986                                 split++;
3987                         action_n++;
3988                         break;
3989                 default:
3990                         action_n++;
3991                         break;
3992                 }
3993         }
3994         if (split && queue_action)
3995                 return action_n;
3996         return 0;
3997 }
3998
3999 /* Declare flow create/destroy prototype in advance. */
4000 static uint32_t
4001 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
4002                  const struct rte_flow_attr *attr,
4003                  const struct rte_flow_item items[],
4004                  const struct rte_flow_action actions[],
4005                  bool external, struct rte_flow_error *error);
4006
4007 static void
4008 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
4009                   uint32_t flow_idx);
4010
4011 int
4012 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
4013                       struct mlx5_hlist_entry *entry,
4014                       uint64_t key, void *cb_ctx __rte_unused)
4015 {
4016         struct mlx5_flow_mreg_copy_resource *mcp_res =
4017                 container_of(entry, typeof(*mcp_res), hlist_ent);
4018
4019         return mcp_res->mark_id != key;
4020 }
4021
4022 struct mlx5_hlist_entry *
4023 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
4024                        void *cb_ctx)
4025 {
4026         struct rte_eth_dev *dev = list->ctx;
4027         struct mlx5_priv *priv = dev->data->dev_private;
4028         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4029         struct mlx5_flow_mreg_copy_resource *mcp_res;
4030         struct rte_flow_error *error = ctx->error;
4031         uint32_t idx = 0;
4032         int ret;
4033         uint32_t mark_id = key;
4034         struct rte_flow_attr attr = {
4035                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4036                 .ingress = 1,
4037         };
4038         struct mlx5_rte_flow_item_tag tag_spec = {
4039                 .data = mark_id,
4040         };
4041         struct rte_flow_item items[] = {
4042                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4043         };
4044         struct rte_flow_action_mark ftag = {
4045                 .id = mark_id,
4046         };
4047         struct mlx5_flow_action_copy_mreg cp_mreg = {
4048                 .dst = REG_B,
4049                 .src = REG_NON,
4050         };
4051         struct rte_flow_action_jump jump = {
4052                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4053         };
4054         struct rte_flow_action actions[] = {
4055                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4056         };
4057
4058         /* Fill the register fileds in the flow. */
4059         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4060         if (ret < 0)
4061                 return NULL;
4062         tag_spec.id = ret;
4063         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4064         if (ret < 0)
4065                 return NULL;
4066         cp_mreg.src = ret;
4067         /* Provide the full width of FLAG specific value. */
4068         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4069                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4070         /* Build a new flow. */
4071         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4072                 items[0] = (struct rte_flow_item){
4073                         .type = (enum rte_flow_item_type)
4074                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4075                         .spec = &tag_spec,
4076                 };
4077                 items[1] = (struct rte_flow_item){
4078                         .type = RTE_FLOW_ITEM_TYPE_END,
4079                 };
4080                 actions[0] = (struct rte_flow_action){
4081                         .type = (enum rte_flow_action_type)
4082                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4083                         .conf = &ftag,
4084                 };
4085                 actions[1] = (struct rte_flow_action){
4086                         .type = (enum rte_flow_action_type)
4087                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4088                         .conf = &cp_mreg,
4089                 };
4090                 actions[2] = (struct rte_flow_action){
4091                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4092                         .conf = &jump,
4093                 };
4094                 actions[3] = (struct rte_flow_action){
4095                         .type = RTE_FLOW_ACTION_TYPE_END,
4096                 };
4097         } else {
4098                 /* Default rule, wildcard match. */
4099                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4100                 items[0] = (struct rte_flow_item){
4101                         .type = RTE_FLOW_ITEM_TYPE_END,
4102                 };
4103                 actions[0] = (struct rte_flow_action){
4104                         .type = (enum rte_flow_action_type)
4105                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4106                         .conf = &cp_mreg,
4107                 };
4108                 actions[1] = (struct rte_flow_action){
4109                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4110                         .conf = &jump,
4111                 };
4112                 actions[2] = (struct rte_flow_action){
4113                         .type = RTE_FLOW_ACTION_TYPE_END,
4114                 };
4115         }
4116         /* Build a new entry. */
4117         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4118         if (!mcp_res) {
4119                 rte_errno = ENOMEM;
4120                 return NULL;
4121         }
4122         mcp_res->idx = idx;
4123         mcp_res->mark_id = mark_id;
4124         /*
4125          * The copy Flows are not included in any list. There
4126          * ones are referenced from other Flows and can not
4127          * be applied, removed, deleted in ardbitrary order
4128          * by list traversing.
4129          */
4130         mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
4131                                          actions, false, error);
4132         if (!mcp_res->rix_flow) {
4133                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4134                 return NULL;
4135         }
4136         return &mcp_res->hlist_ent;
4137 }
4138
4139 /**
4140  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4141  *
4142  * As mark_id is unique, if there's already a registered flow for the mark_id,
4143  * return by increasing the reference counter of the resource. Otherwise, create
4144  * the resource (mcp_res) and flow.
4145  *
4146  * Flow looks like,
4147  *   - If ingress port is ANY and reg_c[1] is mark_id,
4148  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4149  *
4150  * For default flow (zero mark_id), flow is like,
4151  *   - If ingress port is ANY,
4152  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4153  *
4154  * @param dev
4155  *   Pointer to Ethernet device.
4156  * @param mark_id
4157  *   ID of MARK action, zero means default flow for META.
4158  * @param[out] error
4159  *   Perform verbose error reporting if not NULL.
4160  *
4161  * @return
4162  *   Associated resource on success, NULL otherwise and rte_errno is set.
4163  */
4164 static struct mlx5_flow_mreg_copy_resource *
4165 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4166                           struct rte_flow_error *error)
4167 {
4168         struct mlx5_priv *priv = dev->data->dev_private;
4169         struct mlx5_hlist_entry *entry;
4170         struct mlx5_flow_cb_ctx ctx = {
4171                 .dev = dev,
4172                 .error = error,
4173         };
4174
4175         /* Check if already registered. */
4176         MLX5_ASSERT(priv->mreg_cp_tbl);
4177         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4178         if (!entry)
4179                 return NULL;
4180         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4181                             hlist_ent);
4182 }
4183
4184 void
4185 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
4186 {
4187         struct mlx5_flow_mreg_copy_resource *mcp_res =
4188                 container_of(entry, typeof(*mcp_res), hlist_ent);
4189         struct rte_eth_dev *dev = list->ctx;
4190         struct mlx5_priv *priv = dev->data->dev_private;
4191
4192         MLX5_ASSERT(mcp_res->rix_flow);
4193         flow_list_destroy(dev, NULL, mcp_res->rix_flow);
4194         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4195 }
4196
4197 /**
4198  * Release flow in RX_CP_TBL.
4199  *
4200  * @param dev
4201  *   Pointer to Ethernet device.
4202  * @flow
4203  *   Parent flow for wich copying is provided.
4204  */
4205 static void
4206 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4207                           struct rte_flow *flow)
4208 {
4209         struct mlx5_flow_mreg_copy_resource *mcp_res;
4210         struct mlx5_priv *priv = dev->data->dev_private;
4211
4212         if (!flow->rix_mreg_copy)
4213                 return;
4214         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4215                                  flow->rix_mreg_copy);
4216         if (!mcp_res || !priv->mreg_cp_tbl)
4217                 return;
4218         MLX5_ASSERT(mcp_res->rix_flow);
4219         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4220         flow->rix_mreg_copy = 0;
4221 }
4222
4223 /**
4224  * Remove the default copy action from RX_CP_TBL.
4225  *
4226  * This functions is called in the mlx5_dev_start(). No thread safe
4227  * is guaranteed.
4228  *
4229  * @param dev
4230  *   Pointer to Ethernet device.
4231  */
4232 static void
4233 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4234 {
4235         struct mlx5_hlist_entry *entry;
4236         struct mlx5_priv *priv = dev->data->dev_private;
4237
4238         /* Check if default flow is registered. */
4239         if (!priv->mreg_cp_tbl)
4240                 return;
4241         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
4242                                   MLX5_DEFAULT_COPY_ID, NULL);
4243         if (!entry)
4244                 return;
4245         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4246 }
4247
4248 /**
4249  * Add the default copy action in in RX_CP_TBL.
4250  *
4251  * This functions is called in the mlx5_dev_start(). No thread safe
4252  * is guaranteed.
4253  *
4254  * @param dev
4255  *   Pointer to Ethernet device.
4256  * @param[out] error
4257  *   Perform verbose error reporting if not NULL.
4258  *
4259  * @return
4260  *   0 for success, negative value otherwise and rte_errno is set.
4261  */
4262 static int
4263 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4264                                   struct rte_flow_error *error)
4265 {
4266         struct mlx5_priv *priv = dev->data->dev_private;
4267         struct mlx5_flow_mreg_copy_resource *mcp_res;
4268
4269         /* Check whether extensive metadata feature is engaged. */
4270         if (!priv->config.dv_flow_en ||
4271             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4272             !mlx5_flow_ext_mreg_supported(dev) ||
4273             !priv->sh->dv_regc0_mask)
4274                 return 0;
4275         /*
4276          * Add default mreg copy flow may be called multiple time, but
4277          * only be called once in stop. Avoid register it twice.
4278          */
4279         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
4280                 return 0;
4281         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
4282         if (!mcp_res)
4283                 return -rte_errno;
4284         return 0;
4285 }
4286
4287 /**
4288  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4289  *
4290  * All the flow having Q/RSS action should be split by
4291  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4292  * performs the following,
4293  *   - CQE->flow_tag := reg_c[1] (MARK)
4294  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4295  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4296  * but there should be a flow per each MARK ID set by MARK action.
4297  *
4298  * For the aforementioned reason, if there's a MARK action in flow's action
4299  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4300  * the MARK ID to CQE's flow_tag like,
4301  *   - If reg_c[1] is mark_id,
4302  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4303  *
4304  * For SET_META action which stores value in reg_c[0], as the destination is
4305  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4306  * MARK ID means the default flow. The default flow looks like,
4307  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4308  *
4309  * @param dev
4310  *   Pointer to Ethernet device.
4311  * @param flow
4312  *   Pointer to flow structure.
4313  * @param[in] actions
4314  *   Pointer to the list of actions.
4315  * @param[out] error
4316  *   Perform verbose error reporting if not NULL.
4317  *
4318  * @return
4319  *   0 on success, negative value otherwise and rte_errno is set.
4320  */
4321 static int
4322 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4323                             struct rte_flow *flow,
4324                             const struct rte_flow_action *actions,
4325                             struct rte_flow_error *error)
4326 {
4327         struct mlx5_priv *priv = dev->data->dev_private;
4328         struct mlx5_dev_config *config = &priv->config;
4329         struct mlx5_flow_mreg_copy_resource *mcp_res;
4330         const struct rte_flow_action_mark *mark;
4331
4332         /* Check whether extensive metadata feature is engaged. */
4333         if (!config->dv_flow_en ||
4334             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4335             !mlx5_flow_ext_mreg_supported(dev) ||
4336             !priv->sh->dv_regc0_mask)
4337                 return 0;
4338         /* Find MARK action. */
4339         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4340                 switch (actions->type) {
4341                 case RTE_FLOW_ACTION_TYPE_FLAG:
4342                         mcp_res = flow_mreg_add_copy_action
4343                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4344                         if (!mcp_res)
4345                                 return -rte_errno;
4346                         flow->rix_mreg_copy = mcp_res->idx;
4347                         return 0;
4348                 case RTE_FLOW_ACTION_TYPE_MARK:
4349                         mark = (const struct rte_flow_action_mark *)
4350                                 actions->conf;
4351                         mcp_res =
4352                                 flow_mreg_add_copy_action(dev, mark->id, error);
4353                         if (!mcp_res)
4354                                 return -rte_errno;
4355                         flow->rix_mreg_copy = mcp_res->idx;
4356                         return 0;
4357                 default:
4358                         break;
4359                 }
4360         }
4361         return 0;
4362 }
4363
4364 #define MLX5_MAX_SPLIT_ACTIONS 24
4365 #define MLX5_MAX_SPLIT_ITEMS 24
4366
4367 /**
4368  * Split the hairpin flow.
4369  * Since HW can't support encap and push-vlan on Rx, we move these
4370  * actions to Tx.
4371  * If the count action is after the encap then we also
4372  * move the count action. in this case the count will also measure
4373  * the outer bytes.
4374  *
4375  * @param dev
4376  *   Pointer to Ethernet device.
4377  * @param[in] actions
4378  *   Associated actions (list terminated by the END action).
4379  * @param[out] actions_rx
4380  *   Rx flow actions.
4381  * @param[out] actions_tx
4382  *   Tx flow actions..
4383  * @param[out] pattern_tx
4384  *   The pattern items for the Tx flow.
4385  * @param[out] flow_id
4386  *   The flow ID connected to this flow.
4387  *
4388  * @return
4389  *   0 on success.
4390  */
4391 static int
4392 flow_hairpin_split(struct rte_eth_dev *dev,
4393                    const struct rte_flow_action actions[],
4394                    struct rte_flow_action actions_rx[],
4395                    struct rte_flow_action actions_tx[],
4396                    struct rte_flow_item pattern_tx[],
4397                    uint32_t flow_id)
4398 {
4399         const struct rte_flow_action_raw_encap *raw_encap;
4400         const struct rte_flow_action_raw_decap *raw_decap;
4401         struct mlx5_rte_flow_action_set_tag *set_tag;
4402         struct rte_flow_action *tag_action;
4403         struct mlx5_rte_flow_item_tag *tag_item;
4404         struct rte_flow_item *item;
4405         char *addr;
4406         int encap = 0;
4407
4408         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4409                 switch (actions->type) {
4410                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4411                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4412                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4413                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4414                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4415                         rte_memcpy(actions_tx, actions,
4416                                sizeof(struct rte_flow_action));
4417                         actions_tx++;
4418                         break;
4419                 case RTE_FLOW_ACTION_TYPE_COUNT:
4420                         if (encap) {
4421                                 rte_memcpy(actions_tx, actions,
4422                                            sizeof(struct rte_flow_action));
4423                                 actions_tx++;
4424                         } else {
4425                                 rte_memcpy(actions_rx, actions,
4426                                            sizeof(struct rte_flow_action));
4427                                 actions_rx++;
4428                         }
4429                         break;
4430                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4431                         raw_encap = actions->conf;
4432                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4433                                 memcpy(actions_tx, actions,
4434                                        sizeof(struct rte_flow_action));
4435                                 actions_tx++;
4436                                 encap = 1;
4437                         } else {
4438                                 rte_memcpy(actions_rx, actions,
4439                                            sizeof(struct rte_flow_action));
4440                                 actions_rx++;
4441                         }
4442                         break;
4443                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4444                         raw_decap = actions->conf;
4445                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4446                                 memcpy(actions_tx, actions,
4447                                        sizeof(struct rte_flow_action));
4448                                 actions_tx++;
4449                         } else {
4450                                 rte_memcpy(actions_rx, actions,
4451                                            sizeof(struct rte_flow_action));
4452                                 actions_rx++;
4453                         }
4454                         break;
4455                 default:
4456                         rte_memcpy(actions_rx, actions,
4457                                    sizeof(struct rte_flow_action));
4458                         actions_rx++;
4459                         break;
4460                 }
4461         }
4462         /* Add set meta action and end action for the Rx flow. */
4463         tag_action = actions_rx;
4464         tag_action->type = (enum rte_flow_action_type)
4465                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4466         actions_rx++;
4467         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4468         actions_rx++;
4469         set_tag = (void *)actions_rx;
4470         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4471                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4472                 .data = flow_id,
4473         };
4474         MLX5_ASSERT(set_tag->id > REG_NON);
4475         tag_action->conf = set_tag;
4476         /* Create Tx item list. */
4477         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4478         addr = (void *)&pattern_tx[2];
4479         item = pattern_tx;
4480         item->type = (enum rte_flow_item_type)
4481                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4482         tag_item = (void *)addr;
4483         tag_item->data = flow_id;
4484         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4485         MLX5_ASSERT(set_tag->id > REG_NON);
4486         item->spec = tag_item;
4487         addr += sizeof(struct mlx5_rte_flow_item_tag);
4488         tag_item = (void *)addr;
4489         tag_item->data = UINT32_MAX;
4490         tag_item->id = UINT16_MAX;
4491         item->mask = tag_item;
4492         item->last = NULL;
4493         item++;
4494         item->type = RTE_FLOW_ITEM_TYPE_END;
4495         return 0;
4496 }
4497
4498 /**
4499  * The last stage of splitting chain, just creates the subflow
4500  * without any modification.
4501  *
4502  * @param[in] dev
4503  *   Pointer to Ethernet device.
4504  * @param[in] flow
4505  *   Parent flow structure pointer.
4506  * @param[in, out] sub_flow
4507  *   Pointer to return the created subflow, may be NULL.
4508  * @param[in] attr
4509  *   Flow rule attributes.
4510  * @param[in] items
4511  *   Pattern specification (list terminated by the END pattern item).
4512  * @param[in] actions
4513  *   Associated actions (list terminated by the END action).
4514  * @param[in] flow_split_info
4515  *   Pointer to flow split info structure.
4516  * @param[out] error
4517  *   Perform verbose error reporting if not NULL.
4518  * @return
4519  *   0 on success, negative value otherwise
4520  */
4521 static int
4522 flow_create_split_inner(struct rte_eth_dev *dev,
4523                         struct rte_flow *flow,
4524                         struct mlx5_flow **sub_flow,
4525                         const struct rte_flow_attr *attr,
4526                         const struct rte_flow_item items[],
4527                         const struct rte_flow_action actions[],
4528                         struct mlx5_flow_split_info *flow_split_info,
4529                         struct rte_flow_error *error)
4530 {
4531         struct mlx5_flow *dev_flow;
4532
4533         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4534                                     flow_split_info->flow_idx, error);
4535         if (!dev_flow)
4536                 return -rte_errno;
4537         dev_flow->flow = flow;
4538         dev_flow->external = flow_split_info->external;
4539         dev_flow->skip_scale = flow_split_info->skip_scale;
4540         /* Subflow object was created, we must include one in the list. */
4541         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4542                       dev_flow->handle, next);
4543         /*
4544          * If dev_flow is as one of the suffix flow, some actions in suffix
4545          * flow may need some user defined item layer flags, and pass the
4546          * Metadate rxq mark flag to suffix flow as well.
4547          */
4548         if (flow_split_info->prefix_layers)
4549                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4550         if (flow_split_info->prefix_mark)
4551                 dev_flow->handle->mark = 1;
4552         if (sub_flow)
4553                 *sub_flow = dev_flow;
4554 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4555         dev_flow->dv.table_id = flow_split_info->table_id;
4556 #endif
4557         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4558 }
4559
4560 /**
4561  * Get the sub policy of a meter.
4562  *
4563  * @param[in] dev
4564  *   Pointer to Ethernet device.
4565  * @param[in] flow
4566  *   Parent flow structure pointer.
4567  * @param[in] policy_id;
4568  *   Meter Policy id.
4569  * @param[in] attr
4570  *   Flow rule attributes.
4571  * @param[in] items
4572  *   Pattern specification (list terminated by the END pattern item).
4573  * @param[out] error
4574  *   Perform verbose error reporting if not NULL.
4575  *
4576  * @return
4577  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4578  */
4579 static struct mlx5_flow_meter_sub_policy *
4580 get_meter_sub_policy(struct rte_eth_dev *dev,
4581                      struct rte_flow *flow,
4582                      uint32_t policy_id,
4583                      const struct rte_flow_attr *attr,
4584                      const struct rte_flow_item items[],
4585                      struct rte_flow_error *error)
4586 {
4587         struct mlx5_flow_meter_policy *policy;
4588         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4589
4590         policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL);
4591         if (!policy) {
4592                 rte_flow_error_set(error, EINVAL,
4593                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4594                                    "Failed to find Meter Policy.");
4595                 goto exit;
4596         }
4597         if (policy->is_rss ||
4598                 (policy->is_queue &&
4599         !policy->sub_policys[MLX5_MTR_DOMAIN_INGRESS][0]->rix_hrxq[0])) {
4600                 struct mlx5_flow_workspace *wks =
4601                                 mlx5_flow_get_thread_workspace();
4602                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4603                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4604                 uint32_t i;
4605
4606                 MLX5_ASSERT(wks);
4607                 /**
4608                  * This is a tmp dev_flow,
4609                  * no need to register any matcher for it in translate.
4610                  */
4611                 wks->skip_matcher_reg = 1;
4612                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4613                         struct mlx5_flow dev_flow = {0};
4614                         struct mlx5_flow_handle dev_handle = { {0} };
4615
4616                         rss_desc_v[i] = wks->rss_desc;
4617                         if (policy->is_rss) {
4618                                 const void *rss_act =
4619                                         policy->act_cnt[i].rss->conf;
4620                                 struct rte_flow_action rss_actions[2] = {
4621                                         [0] = {
4622                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4623                                         .conf = rss_act
4624                                         },
4625                                         [1] = {
4626                                         .type = RTE_FLOW_ACTION_TYPE_END,
4627                                         .conf = NULL
4628                                         }
4629                                 };
4630
4631                                 dev_flow.handle = &dev_handle;
4632                                 dev_flow.ingress = attr->ingress;
4633                                 dev_flow.flow = flow;
4634                                 dev_flow.external = 0;
4635 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4636                                 dev_flow.dv.transfer = attr->transfer;
4637 #endif
4638                                 /**
4639                                  * Translate RSS action to get rss hash fields.
4640                                  */
4641                                 if (flow_drv_translate(dev, &dev_flow, attr,
4642                                                 items, rss_actions, error))
4643                                         goto exit;
4644                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4645                                 rss_desc_v[i].hash_fields =
4646                                                 dev_flow.hash_fields;
4647                                 rss_desc_v[i].queue_num =
4648                                                 rss_desc_v[i].hash_fields ?
4649                                                 rss_desc_v[i].queue_num : 1;
4650                         } else {
4651                                 /* This is queue action. */
4652                                 rss_desc_v[i].key_len = 0;
4653                                 rss_desc_v[i].hash_fields = 0;
4654                                 rss_desc_v[i].queue =
4655                                         &policy->act_cnt[i].queue;
4656                                 rss_desc_v[i].queue_num = 1;
4657                         }
4658                         rss_desc[i] = &rss_desc_v[i];
4659                 }
4660                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4661                                                 flow, policy, rss_desc);
4662         } else {
4663                 enum mlx5_meter_domain mtr_domain =
4664                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4665                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4666                                         MLX5_MTR_DOMAIN_INGRESS;
4667                 sub_policy = policy->sub_policys[mtr_domain][0];
4668         }
4669         if (!sub_policy) {
4670                 rte_flow_error_set(error, EINVAL,
4671                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4672                         "Failed to get meter sub-policy.");
4673                 goto exit;
4674         }
4675 exit:
4676         return sub_policy;
4677 }
4678
4679 /**
4680  * Split the meter flow.
4681  *
4682  * As meter flow will split to three sub flow, other than meter
4683  * action, the other actions make sense to only meter accepts
4684  * the packet. If it need to be dropped, no other additional
4685  * actions should be take.
4686  *
4687  * One kind of special action which decapsulates the L3 tunnel
4688  * header will be in the prefix sub flow, as not to take the
4689  * L3 tunnel header into account.
4690  *
4691  * @param[in] dev
4692  *   Pointer to Ethernet device.
4693  * @param[in] flow
4694  *   Parent flow structure pointer.
4695  * @param[in] fm
4696  *   Pointer to flow meter structure.
4697  * @param[in] attr
4698  *   Flow rule attributes.
4699  * @param[in] items
4700  *   Pattern specification (list terminated by the END pattern item).
4701  * @param[out] sfx_items
4702  *   Suffix flow match items (list terminated by the END pattern item).
4703  * @param[in] actions
4704  *   Associated actions (list terminated by the END action).
4705  * @param[out] actions_sfx
4706  *   Suffix flow actions.
4707  * @param[out] actions_pre
4708  *   Prefix flow actions.
4709  * @param[out] mtr_flow_id
4710  *   Pointer to meter flow id.
4711  * @param[out] error
4712  *   Perform verbose error reporting if not NULL.
4713  *
4714  * @return
4715  *   0 on success, a negative errno value otherwise and rte_errno is set.
4716  */
4717 static int
4718 flow_meter_split_prep(struct rte_eth_dev *dev,
4719                       struct rte_flow *flow,
4720                       struct mlx5_flow_meter_info *fm,
4721                       const struct rte_flow_attr *attr,
4722                       const struct rte_flow_item items[],
4723                       struct rte_flow_item sfx_items[],
4724                       const struct rte_flow_action actions[],
4725                       struct rte_flow_action actions_sfx[],
4726                       struct rte_flow_action actions_pre[],
4727                       uint32_t *mtr_flow_id,
4728                       struct rte_flow_error *error)
4729 {
4730         struct mlx5_priv *priv = dev->data->dev_private;
4731         struct rte_flow_action *tag_action = NULL;
4732         struct rte_flow_item *tag_item;
4733         struct mlx5_rte_flow_action_set_tag *set_tag;
4734         const struct rte_flow_action_raw_encap *raw_encap;
4735         const struct rte_flow_action_raw_decap *raw_decap;
4736         struct mlx5_rte_flow_item_tag *tag_item_spec;
4737         struct mlx5_rte_flow_item_tag *tag_item_mask;
4738         uint32_t tag_id = 0;
4739         bool copy_vlan = false;
4740         struct rte_flow_action *hw_mtr_action;
4741         struct rte_flow_action *action_pre_head = NULL;
4742         bool mtr_first = priv->sh->meter_aso_en &&
4743                         (attr->egress ||
4744                         (attr->transfer && priv->representor_id != UINT16_MAX));
4745         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4746         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4747                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4748         uint32_t flow_id = 0;
4749         uint32_t flow_id_reversed = 0;
4750         uint8_t flow_id_bits = 0;
4751         int shift;
4752
4753         /* For ASO meter, meter must be before tag in TX direction. */
4754         if (mtr_first) {
4755                 action_pre_head = actions_pre++;
4756                 /* Leave space for tag action. */
4757                 tag_action = actions_pre++;
4758         }
4759         /* Prepare the actions for prefix and suffix flow. */
4760         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4761                 struct rte_flow_action *action_cur = NULL;
4762
4763                 switch (actions->type) {
4764                 case RTE_FLOW_ACTION_TYPE_METER:
4765                         if (mtr_first) {
4766                                 action_cur = action_pre_head;
4767                         } else {
4768                                 /* Leave space for tag action. */
4769                                 tag_action = actions_pre++;
4770                                 action_cur = actions_pre++;
4771                         }
4772                         break;
4773                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4774                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4775                         action_cur = actions_pre++;
4776                         break;
4777                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4778                         raw_encap = actions->conf;
4779                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4780                                 action_cur = actions_pre++;
4781                         break;
4782                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4783                         raw_decap = actions->conf;
4784                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4785                                 action_cur = actions_pre++;
4786                         break;
4787                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4788                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4789                         copy_vlan = true;
4790                         break;
4791                 default:
4792                         break;
4793                 }
4794                 if (!action_cur)
4795                         action_cur = (fm->def_policy) ?
4796                                         actions_sfx++ : actions_pre++;
4797                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4798         }
4799         /* Add end action to the actions. */
4800         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4801         if (priv->sh->meter_aso_en) {
4802                 /**
4803                  * For ASO meter, need to add an extra jump action explicitly,
4804                  * to jump from meter to policer table.
4805                  */
4806                 struct mlx5_flow_meter_sub_policy *sub_policy;
4807                 struct mlx5_flow_tbl_data_entry *tbl_data;
4808
4809                 if (!fm->def_policy) {
4810                         sub_policy = get_meter_sub_policy(dev, flow,
4811                                                           fm->policy_id, attr,
4812                                                           items, error);
4813                         if (!sub_policy)
4814                                 return -rte_errno;
4815                 } else {
4816                         enum mlx5_meter_domain mtr_domain =
4817                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4818                                 attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4819                                         MLX5_MTR_DOMAIN_INGRESS;
4820
4821                         sub_policy =
4822                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
4823                 }
4824                 tbl_data = container_of(sub_policy->tbl_rsc,
4825                                         struct mlx5_flow_tbl_data_entry, tbl);
4826                 hw_mtr_action = actions_pre++;
4827                 hw_mtr_action->type = (enum rte_flow_action_type)
4828                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
4829                 hw_mtr_action->conf = tbl_data->jump.action;
4830         }
4831         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4832         actions_pre++;
4833         if (!tag_action)
4834                 return rte_flow_error_set(error, ENOMEM,
4835                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4836                                         "No tag action space.");
4837         if (!mtr_flow_id) {
4838                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
4839                 goto exit;
4840         }
4841         /* Only default-policy Meter creates mtr flow id. */
4842         if (fm->def_policy) {
4843                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
4844                 if (!tag_id)
4845                         return rte_flow_error_set(error, ENOMEM,
4846                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4847                                         "Failed to allocate meter flow id.");
4848                 flow_id = tag_id - 1;
4849                 flow_id_bits = (!flow_id) ? 1 :
4850                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
4851                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
4852                     mtr_reg_bits) {
4853                         mlx5_ipool_free(fm->flow_ipool, tag_id);
4854                         return rte_flow_error_set(error, EINVAL,
4855                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4856                                         "Meter flow id exceeds max limit.");
4857                 }
4858                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
4859                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
4860         }
4861         /* Prepare the suffix subflow items. */
4862         tag_item = sfx_items++;
4863         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4864                 int item_type = items->type;
4865
4866                 switch (item_type) {
4867                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4868                         memcpy(sfx_items, items, sizeof(*sfx_items));
4869                         sfx_items++;
4870                         break;
4871                 case RTE_FLOW_ITEM_TYPE_VLAN:
4872                         if (copy_vlan) {
4873                                 memcpy(sfx_items, items, sizeof(*sfx_items));
4874                                 /*
4875                                  * Convert to internal match item, it is used
4876                                  * for vlan push and set vid.
4877                                  */
4878                                 sfx_items->type = (enum rte_flow_item_type)
4879                                                   MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4880                                 sfx_items++;
4881                         }
4882                         break;
4883                 default:
4884                         break;
4885                 }
4886         }
4887         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4888         sfx_items++;
4889         /* Build tag actions and items for meter_id/meter flow_id. */
4890         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
4891         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
4892         tag_item_mask = tag_item_spec + 1;
4893         /* Both flow_id and meter_id share the same register. */
4894         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4895                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
4896                                                             0, error),
4897                 .offset = mtr_id_offset,
4898                 .length = mtr_reg_bits,
4899                 .data = flow->meter,
4900         };
4901         /*
4902          * The color Reg bits used by flow_id are growing from
4903          * msb to lsb, so must do bit reverse for flow_id val in RegC.
4904          */
4905         for (shift = 0; shift < flow_id_bits; shift++)
4906                 flow_id_reversed = (flow_id_reversed << 1) |
4907                                 ((flow_id >> shift) & 0x1);
4908         set_tag->data |=
4909                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
4910         tag_item_spec->id = set_tag->id;
4911         tag_item_spec->data = set_tag->data << mtr_id_offset;
4912         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
4913         tag_action->type = (enum rte_flow_action_type)
4914                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4915         tag_action->conf = set_tag;
4916         tag_item->type = (enum rte_flow_item_type)
4917                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4918         tag_item->spec = tag_item_spec;
4919         tag_item->last = NULL;
4920         tag_item->mask = tag_item_mask;
4921 exit:
4922         if (mtr_flow_id)
4923                 *mtr_flow_id = tag_id;
4924         return 0;
4925 }
4926
4927 /**
4928  * Split action list having QUEUE/RSS for metadata register copy.
4929  *
4930  * Once Q/RSS action is detected in user's action list, the flow action
4931  * should be split in order to copy metadata registers, which will happen in
4932  * RX_CP_TBL like,
4933  *   - CQE->flow_tag := reg_c[1] (MARK)
4934  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4935  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
4936  * This is because the last action of each flow must be a terminal action
4937  * (QUEUE, RSS or DROP).
4938  *
4939  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
4940  * stored and kept in the mlx5_flow structure per each sub_flow.
4941  *
4942  * The Q/RSS action is replaced with,
4943  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
4944  * And the following JUMP action is added at the end,
4945  *   - JUMP, to RX_CP_TBL.
4946  *
4947  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
4948  * flow_create_split_metadata() routine. The flow will look like,
4949  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
4950  *
4951  * @param dev
4952  *   Pointer to Ethernet device.
4953  * @param[out] split_actions
4954  *   Pointer to store split actions to jump to CP_TBL.
4955  * @param[in] actions
4956  *   Pointer to the list of original flow actions.
4957  * @param[in] qrss
4958  *   Pointer to the Q/RSS action.
4959  * @param[in] actions_n
4960  *   Number of original actions.
4961  * @param[out] error
4962  *   Perform verbose error reporting if not NULL.
4963  *
4964  * @return
4965  *   non-zero unique flow_id on success, otherwise 0 and
4966  *   error/rte_error are set.
4967  */
4968 static uint32_t
4969 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
4970                           struct rte_flow_action *split_actions,
4971                           const struct rte_flow_action *actions,
4972                           const struct rte_flow_action *qrss,
4973                           int actions_n, struct rte_flow_error *error)
4974 {
4975         struct mlx5_priv *priv = dev->data->dev_private;
4976         struct mlx5_rte_flow_action_set_tag *set_tag;
4977         struct rte_flow_action_jump *jump;
4978         const int qrss_idx = qrss - actions;
4979         uint32_t flow_id = 0;
4980         int ret = 0;
4981
4982         /*
4983          * Given actions will be split
4984          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
4985          * - Add jump to mreg CP_TBL.
4986          * As a result, there will be one more action.
4987          */
4988         ++actions_n;
4989         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
4990         set_tag = (void *)(split_actions + actions_n);
4991         /*
4992          * If tag action is not set to void(it means we are not the meter
4993          * suffix flow), add the tag action. Since meter suffix flow already
4994          * has the tag added.
4995          */
4996         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
4997                 /*
4998                  * Allocate the new subflow ID. This one is unique within
4999                  * device and not shared with representors. Otherwise,
5000                  * we would have to resolve multi-thread access synch
5001                  * issue. Each flow on the shared device is appended
5002                  * with source vport identifier, so the resulting
5003                  * flows will be unique in the shared (by master and
5004                  * representors) domain even if they have coinciding
5005                  * IDs.
5006                  */
5007                 mlx5_ipool_malloc(priv->sh->ipool
5008                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5009                 if (!flow_id)
5010                         return rte_flow_error_set(error, ENOMEM,
5011                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5012                                                   NULL, "can't allocate id "
5013                                                   "for split Q/RSS subflow");
5014                 /* Internal SET_TAG action to set flow ID. */
5015                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5016                         .data = flow_id,
5017                 };
5018                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5019                 if (ret < 0)
5020                         return ret;
5021                 set_tag->id = ret;
5022                 /* Construct new actions array. */
5023                 /* Replace QUEUE/RSS action. */
5024                 split_actions[qrss_idx] = (struct rte_flow_action){
5025                         .type = (enum rte_flow_action_type)
5026                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5027                         .conf = set_tag,
5028                 };
5029         }
5030         /* JUMP action to jump to mreg copy table (CP_TBL). */
5031         jump = (void *)(set_tag + 1);
5032         *jump = (struct rte_flow_action_jump){
5033                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5034         };
5035         split_actions[actions_n - 2] = (struct rte_flow_action){
5036                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5037                 .conf = jump,
5038         };
5039         split_actions[actions_n - 1] = (struct rte_flow_action){
5040                 .type = RTE_FLOW_ACTION_TYPE_END,
5041         };
5042         return flow_id;
5043 }
5044
5045 /**
5046  * Extend the given action list for Tx metadata copy.
5047  *
5048  * Copy the given action list to the ext_actions and add flow metadata register
5049  * copy action in order to copy reg_a set by WQE to reg_c[0].
5050  *
5051  * @param[out] ext_actions
5052  *   Pointer to the extended action list.
5053  * @param[in] actions
5054  *   Pointer to the list of actions.
5055  * @param[in] actions_n
5056  *   Number of actions in the list.
5057  * @param[out] error
5058  *   Perform verbose error reporting if not NULL.
5059  * @param[in] encap_idx
5060  *   The encap action inndex.
5061  *
5062  * @return
5063  *   0 on success, negative value otherwise
5064  */
5065 static int
5066 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5067                        struct rte_flow_action *ext_actions,
5068                        const struct rte_flow_action *actions,
5069                        int actions_n, struct rte_flow_error *error,
5070                        int encap_idx)
5071 {
5072         struct mlx5_flow_action_copy_mreg *cp_mreg =
5073                 (struct mlx5_flow_action_copy_mreg *)
5074                         (ext_actions + actions_n + 1);
5075         int ret;
5076
5077         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5078         if (ret < 0)
5079                 return ret;
5080         cp_mreg->dst = ret;
5081         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5082         if (ret < 0)
5083                 return ret;
5084         cp_mreg->src = ret;
5085         if (encap_idx != 0)
5086                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5087         if (encap_idx == actions_n - 1) {
5088                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5089                         .type = (enum rte_flow_action_type)
5090                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5091                         .conf = cp_mreg,
5092                 };
5093                 ext_actions[actions_n] = (struct rte_flow_action){
5094                         .type = RTE_FLOW_ACTION_TYPE_END,
5095                 };
5096         } else {
5097                 ext_actions[encap_idx] = (struct rte_flow_action){
5098                         .type = (enum rte_flow_action_type)
5099                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5100                         .conf = cp_mreg,
5101                 };
5102                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5103                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5104         }
5105         return 0;
5106 }
5107
5108 /**
5109  * Check the match action from the action list.
5110  *
5111  * @param[in] actions
5112  *   Pointer to the list of actions.
5113  * @param[in] attr
5114  *   Flow rule attributes.
5115  * @param[in] action
5116  *   The action to be check if exist.
5117  * @param[out] match_action_pos
5118  *   Pointer to the position of the matched action if exists, otherwise is -1.
5119  * @param[out] qrss_action_pos
5120  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5121  * @param[out] modify_after_mirror
5122  *   Pointer to the flag of modify action after FDB mirroring.
5123  *
5124  * @return
5125  *   > 0 the total number of actions.
5126  *   0 if not found match action in action list.
5127  */
5128 static int
5129 flow_check_match_action(const struct rte_flow_action actions[],
5130                         const struct rte_flow_attr *attr,
5131                         enum rte_flow_action_type action,
5132                         int *match_action_pos, int *qrss_action_pos,
5133                         int *modify_after_mirror)
5134 {
5135         const struct rte_flow_action_sample *sample;
5136         int actions_n = 0;
5137         uint32_t ratio = 0;
5138         int sub_type = 0;
5139         int flag = 0;
5140         int fdb_mirror = 0;
5141
5142         *match_action_pos = -1;
5143         *qrss_action_pos = -1;
5144         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5145                 if (actions->type == action) {
5146                         flag = 1;
5147                         *match_action_pos = actions_n;
5148                 }
5149                 switch (actions->type) {
5150                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5151                 case RTE_FLOW_ACTION_TYPE_RSS:
5152                         *qrss_action_pos = actions_n;
5153                         break;
5154                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5155                         sample = actions->conf;
5156                         ratio = sample->ratio;
5157                         sub_type = ((const struct rte_flow_action *)
5158                                         (sample->actions))->type;
5159                         if (ratio == 1 && attr->transfer)
5160                                 fdb_mirror = 1;
5161                         break;
5162                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5163                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5164                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5165                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5166                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5167                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5168                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5169                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5170                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5171                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5172                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5173                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5174                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5175                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5176                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5177                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5178                 case RTE_FLOW_ACTION_TYPE_FLAG:
5179                 case RTE_FLOW_ACTION_TYPE_MARK:
5180                 case RTE_FLOW_ACTION_TYPE_SET_META:
5181                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5182                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5183                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5184                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5185                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5186                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5187                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5188                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5189                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5190                 case RTE_FLOW_ACTION_TYPE_METER:
5191                         if (fdb_mirror)
5192                                 *modify_after_mirror = 1;
5193                         break;
5194                 default:
5195                         break;
5196                 }
5197                 actions_n++;
5198         }
5199         if (flag && fdb_mirror && !*modify_after_mirror) {
5200                 /* FDB mirroring uses the destination array to implement
5201                  * instead of FLOW_SAMPLER object.
5202                  */
5203                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5204                         flag = 0;
5205         }
5206         /* Count RTE_FLOW_ACTION_TYPE_END. */
5207         return flag ? actions_n + 1 : 0;
5208 }
5209
5210 #define SAMPLE_SUFFIX_ITEM 2
5211
5212 /**
5213  * Split the sample flow.
5214  *
5215  * As sample flow will split to two sub flow, sample flow with
5216  * sample action, the other actions will move to new suffix flow.
5217  *
5218  * Also add unique tag id with tag action in the sample flow,
5219  * the same tag id will be as match in the suffix flow.
5220  *
5221  * @param dev
5222  *   Pointer to Ethernet device.
5223  * @param[in] add_tag
5224  *   Add extra tag action flag.
5225  * @param[out] sfx_items
5226  *   Suffix flow match items (list terminated by the END pattern item).
5227  * @param[in] actions
5228  *   Associated actions (list terminated by the END action).
5229  * @param[out] actions_sfx
5230  *   Suffix flow actions.
5231  * @param[out] actions_pre
5232  *   Prefix flow actions.
5233  * @param[in] actions_n
5234  *  The total number of actions.
5235  * @param[in] sample_action_pos
5236  *   The sample action position.
5237  * @param[in] qrss_action_pos
5238  *   The Queue/RSS action position.
5239  * @param[in] jump_table
5240  *   Add extra jump action flag.
5241  * @param[out] error
5242  *   Perform verbose error reporting if not NULL.
5243  *
5244  * @return
5245  *   0 on success, or unique flow_id, a negative errno value
5246  *   otherwise and rte_errno is set.
5247  */
5248 static int
5249 flow_sample_split_prep(struct rte_eth_dev *dev,
5250                        int add_tag,
5251                        struct rte_flow_item sfx_items[],
5252                        const struct rte_flow_action actions[],
5253                        struct rte_flow_action actions_sfx[],
5254                        struct rte_flow_action actions_pre[],
5255                        int actions_n,
5256                        int sample_action_pos,
5257                        int qrss_action_pos,
5258                        int jump_table,
5259                        struct rte_flow_error *error)
5260 {
5261         struct mlx5_priv *priv = dev->data->dev_private;
5262         struct mlx5_rte_flow_action_set_tag *set_tag;
5263         struct mlx5_rte_flow_item_tag *tag_spec;
5264         struct mlx5_rte_flow_item_tag *tag_mask;
5265         struct rte_flow_action_jump *jump_action;
5266         uint32_t tag_id = 0;
5267         int index;
5268         int append_index = 0;
5269         int ret;
5270
5271         if (sample_action_pos < 0)
5272                 return rte_flow_error_set(error, EINVAL,
5273                                           RTE_FLOW_ERROR_TYPE_ACTION,
5274                                           NULL, "invalid position of sample "
5275                                           "action in list");
5276         /* Prepare the actions for prefix and suffix flow. */
5277         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5278                 index = qrss_action_pos;
5279                 /* Put the preceding the Queue/RSS action into prefix flow. */
5280                 if (index != 0)
5281                         memcpy(actions_pre, actions,
5282                                sizeof(struct rte_flow_action) * index);
5283                 /* Put others preceding the sample action into prefix flow. */
5284                 if (sample_action_pos > index + 1)
5285                         memcpy(actions_pre + index, actions + index + 1,
5286                                sizeof(struct rte_flow_action) *
5287                                (sample_action_pos - index - 1));
5288                 index = sample_action_pos - 1;
5289                 /* Put Queue/RSS action into Suffix flow. */
5290                 memcpy(actions_sfx, actions + qrss_action_pos,
5291                        sizeof(struct rte_flow_action));
5292                 actions_sfx++;
5293         } else {
5294                 index = sample_action_pos;
5295                 if (index != 0)
5296                         memcpy(actions_pre, actions,
5297                                sizeof(struct rte_flow_action) * index);
5298         }
5299         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5300          * For CX6DX and above, metadata registers Cx preserve their value,
5301          * add an extra tag action for NIC-RX and E-Switch Domain.
5302          */
5303         if (add_tag) {
5304                 /* Prepare the prefix tag action. */
5305                 append_index++;
5306                 set_tag = (void *)(actions_pre + actions_n + append_index);
5307                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5308                 if (ret < 0)
5309                         return ret;
5310                 mlx5_ipool_malloc(priv->sh->ipool
5311                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5312                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5313                         .id = ret,
5314                         .data = tag_id,
5315                 };
5316                 /* Prepare the suffix subflow items. */
5317                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5318                 tag_spec->data = tag_id;
5319                 tag_spec->id = set_tag->id;
5320                 tag_mask = tag_spec + 1;
5321                 tag_mask->data = UINT32_MAX;
5322                 sfx_items[0] = (struct rte_flow_item){
5323                         .type = (enum rte_flow_item_type)
5324                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5325                         .spec = tag_spec,
5326                         .last = NULL,
5327                         .mask = tag_mask,
5328                 };
5329                 sfx_items[1] = (struct rte_flow_item){
5330                         .type = (enum rte_flow_item_type)
5331                                 RTE_FLOW_ITEM_TYPE_END,
5332                 };
5333                 /* Prepare the tag action in prefix subflow. */
5334                 actions_pre[index++] =
5335                         (struct rte_flow_action){
5336                         .type = (enum rte_flow_action_type)
5337                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5338                         .conf = set_tag,
5339                 };
5340         }
5341         memcpy(actions_pre + index, actions + sample_action_pos,
5342                sizeof(struct rte_flow_action));
5343         index += 1;
5344         /* For the modify action after the sample action in E-Switch mirroring,
5345          * Add the extra jump action in prefix subflow and jump into the next
5346          * table, then do the modify action in the new table.
5347          */
5348         if (jump_table) {
5349                 /* Prepare the prefix jump action. */
5350                 append_index++;
5351                 jump_action = (void *)(actions_pre + actions_n + append_index);
5352                 jump_action->group = jump_table;
5353                 actions_pre[index++] =
5354                         (struct rte_flow_action){
5355                         .type = (enum rte_flow_action_type)
5356                                 RTE_FLOW_ACTION_TYPE_JUMP,
5357                         .conf = jump_action,
5358                 };
5359         }
5360         actions_pre[index] = (struct rte_flow_action){
5361                 .type = (enum rte_flow_action_type)
5362                         RTE_FLOW_ACTION_TYPE_END,
5363         };
5364         /* Put the actions after sample into Suffix flow. */
5365         memcpy(actions_sfx, actions + sample_action_pos + 1,
5366                sizeof(struct rte_flow_action) *
5367                (actions_n - sample_action_pos - 1));
5368         return tag_id;
5369 }
5370
5371 /**
5372  * The splitting for metadata feature.
5373  *
5374  * - Q/RSS action on NIC Rx should be split in order to pass by
5375  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5376  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5377  *
5378  * - All the actions on NIC Tx should have a mreg copy action to
5379  *   copy reg_a from WQE to reg_c[0].
5380  *
5381  * @param dev
5382  *   Pointer to Ethernet device.
5383  * @param[in] flow
5384  *   Parent flow structure pointer.
5385  * @param[in] attr
5386  *   Flow rule attributes.
5387  * @param[in] items
5388  *   Pattern specification (list terminated by the END pattern item).
5389  * @param[in] actions
5390  *   Associated actions (list terminated by the END action).
5391  * @param[in] flow_split_info
5392  *   Pointer to flow split info structure.
5393  * @param[out] error
5394  *   Perform verbose error reporting if not NULL.
5395  * @return
5396  *   0 on success, negative value otherwise
5397  */
5398 static int
5399 flow_create_split_metadata(struct rte_eth_dev *dev,
5400                            struct rte_flow *flow,
5401                            const struct rte_flow_attr *attr,
5402                            const struct rte_flow_item items[],
5403                            const struct rte_flow_action actions[],
5404                            struct mlx5_flow_split_info *flow_split_info,
5405                            struct rte_flow_error *error)
5406 {
5407         struct mlx5_priv *priv = dev->data->dev_private;
5408         struct mlx5_dev_config *config = &priv->config;
5409         const struct rte_flow_action *qrss = NULL;
5410         struct rte_flow_action *ext_actions = NULL;
5411         struct mlx5_flow *dev_flow = NULL;
5412         uint32_t qrss_id = 0;
5413         int mtr_sfx = 0;
5414         size_t act_size;
5415         int actions_n;
5416         int encap_idx;
5417         int ret;
5418
5419         /* Check whether extensive metadata feature is engaged. */
5420         if (!config->dv_flow_en ||
5421             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5422             !mlx5_flow_ext_mreg_supported(dev))
5423                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5424                                                actions, flow_split_info, error);
5425         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5426                                                            &encap_idx);
5427         if (qrss) {
5428                 /* Exclude hairpin flows from splitting. */
5429                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5430                         const struct rte_flow_action_queue *queue;
5431
5432                         queue = qrss->conf;
5433                         if (mlx5_rxq_get_type(dev, queue->index) ==
5434                             MLX5_RXQ_TYPE_HAIRPIN)
5435                                 qrss = NULL;
5436                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5437                         const struct rte_flow_action_rss *rss;
5438
5439                         rss = qrss->conf;
5440                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5441                             MLX5_RXQ_TYPE_HAIRPIN)
5442                                 qrss = NULL;
5443                 }
5444         }
5445         if (qrss) {
5446                 /* Check if it is in meter suffix table. */
5447                 mtr_sfx = attr->group == (attr->transfer ?
5448                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5449                           MLX5_FLOW_TABLE_LEVEL_METER);
5450                 /*
5451                  * Q/RSS action on NIC Rx should be split in order to pass by
5452                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5453                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5454                  */
5455                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5456                            sizeof(struct rte_flow_action_set_tag) +
5457                            sizeof(struct rte_flow_action_jump);
5458                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5459                                           SOCKET_ID_ANY);
5460                 if (!ext_actions)
5461                         return rte_flow_error_set(error, ENOMEM,
5462                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5463                                                   NULL, "no memory to split "
5464                                                   "metadata flow");
5465                 /*
5466                  * If we are the suffix flow of meter, tag already exist.
5467                  * Set the tag action to void.
5468                  */
5469                 if (mtr_sfx)
5470                         ext_actions[qrss - actions].type =
5471                                                 RTE_FLOW_ACTION_TYPE_VOID;
5472                 else
5473                         ext_actions[qrss - actions].type =
5474                                                 (enum rte_flow_action_type)
5475                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5476                 /*
5477                  * Create the new actions list with removed Q/RSS action
5478                  * and appended set tag and jump to register copy table
5479                  * (RX_CP_TBL). We should preallocate unique tag ID here
5480                  * in advance, because it is needed for set tag action.
5481                  */
5482                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5483                                                     qrss, actions_n, error);
5484                 if (!mtr_sfx && !qrss_id) {
5485                         ret = -rte_errno;
5486                         goto exit;
5487                 }
5488         } else if (attr->egress && !attr->transfer) {
5489                 /*
5490                  * All the actions on NIC Tx should have a metadata register
5491                  * copy action to copy reg_a from WQE to reg_c[meta]
5492                  */
5493                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5494                            sizeof(struct mlx5_flow_action_copy_mreg);
5495                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5496                                           SOCKET_ID_ANY);
5497                 if (!ext_actions)
5498                         return rte_flow_error_set(error, ENOMEM,
5499                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5500                                                   NULL, "no memory to split "
5501                                                   "metadata flow");
5502                 /* Create the action list appended with copy register. */
5503                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5504                                              actions_n, error, encap_idx);
5505                 if (ret < 0)
5506                         goto exit;
5507         }
5508         /* Add the unmodified original or prefix subflow. */
5509         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5510                                       items, ext_actions ? ext_actions :
5511                                       actions, flow_split_info, error);
5512         if (ret < 0)
5513                 goto exit;
5514         MLX5_ASSERT(dev_flow);
5515         if (qrss) {
5516                 const struct rte_flow_attr q_attr = {
5517                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5518                         .ingress = 1,
5519                 };
5520                 /* Internal PMD action to set register. */
5521                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5522                         .data = qrss_id,
5523                         .id = REG_NON,
5524                 };
5525                 struct rte_flow_item q_items[] = {
5526                         {
5527                                 .type = (enum rte_flow_item_type)
5528                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5529                                 .spec = &q_tag_spec,
5530                                 .last = NULL,
5531                                 .mask = NULL,
5532                         },
5533                         {
5534                                 .type = RTE_FLOW_ITEM_TYPE_END,
5535                         },
5536                 };
5537                 struct rte_flow_action q_actions[] = {
5538                         {
5539                                 .type = qrss->type,
5540                                 .conf = qrss->conf,
5541                         },
5542                         {
5543                                 .type = RTE_FLOW_ACTION_TYPE_END,
5544                         },
5545                 };
5546                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5547
5548                 /*
5549                  * Configure the tag item only if there is no meter subflow.
5550                  * Since tag is already marked in the meter suffix subflow
5551                  * we can just use the meter suffix items as is.
5552                  */
5553                 if (qrss_id) {
5554                         /* Not meter subflow. */
5555                         MLX5_ASSERT(!mtr_sfx);
5556                         /*
5557                          * Put unique id in prefix flow due to it is destroyed
5558                          * after suffix flow and id will be freed after there
5559                          * is no actual flows with this id and identifier
5560                          * reallocation becomes possible (for example, for
5561                          * other flows in other threads).
5562                          */
5563                         dev_flow->handle->split_flow_id = qrss_id;
5564                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5565                                                    error);
5566                         if (ret < 0)
5567                                 goto exit;
5568                         q_tag_spec.id = ret;
5569                 }
5570                 dev_flow = NULL;
5571                 /* Add suffix subflow to execute Q/RSS. */
5572                 flow_split_info->prefix_layers = layers;
5573                 flow_split_info->prefix_mark = 0;
5574                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5575                                               &q_attr, mtr_sfx ? items :
5576                                               q_items, q_actions,
5577                                               flow_split_info, error);
5578                 if (ret < 0)
5579                         goto exit;
5580                 /* qrss ID should be freed if failed. */
5581                 qrss_id = 0;
5582                 MLX5_ASSERT(dev_flow);
5583         }
5584
5585 exit:
5586         /*
5587          * We do not destroy the partially created sub_flows in case of error.
5588          * These ones are included into parent flow list and will be destroyed
5589          * by flow_drv_destroy.
5590          */
5591         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5592                         qrss_id);
5593         mlx5_free(ext_actions);
5594         return ret;
5595 }
5596
5597 /**
5598  * Create meter internal drop flow with the original pattern.
5599  *
5600  * @param dev
5601  *   Pointer to Ethernet device.
5602  * @param[in] flow
5603  *   Parent flow structure pointer.
5604  * @param[in] attr
5605  *   Flow rule attributes.
5606  * @param[in] items
5607  *   Pattern specification (list terminated by the END pattern item).
5608  * @param[in] flow_split_info
5609  *   Pointer to flow split info structure.
5610  * @param[in] fm
5611  *   Pointer to flow meter structure.
5612  * @param[out] error
5613  *   Perform verbose error reporting if not NULL.
5614  * @return
5615  *   0 on success, negative value otherwise
5616  */
5617 static uint32_t
5618 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5619                         struct rte_flow *flow,
5620                         const struct rte_flow_attr *attr,
5621                         const struct rte_flow_item items[],
5622                         struct mlx5_flow_split_info *flow_split_info,
5623                         struct mlx5_flow_meter_info *fm,
5624                         struct rte_flow_error *error)
5625 {
5626         struct mlx5_flow *dev_flow = NULL;
5627         struct rte_flow_attr drop_attr = *attr;
5628         struct rte_flow_action drop_actions[3];
5629         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5630
5631         MLX5_ASSERT(fm->drop_cnt);
5632         drop_actions[0].type =
5633                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5634         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5635         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5636         drop_actions[1].conf = NULL;
5637         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5638         drop_actions[2].conf = NULL;
5639         drop_split_info.external = false;
5640         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5641         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5642         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5643         return flow_create_split_inner(dev, flow, &dev_flow,
5644                                 &drop_attr, items, drop_actions,
5645                                 &drop_split_info, error);
5646 }
5647
5648 /**
5649  * The splitting for meter feature.
5650  *
5651  * - The meter flow will be split to two flows as prefix and
5652  *   suffix flow. The packets make sense only it pass the prefix
5653  *   meter action.
5654  *
5655  * - Reg_C_5 is used for the packet to match betweend prefix and
5656  *   suffix flow.
5657  *
5658  * @param dev
5659  *   Pointer to Ethernet device.
5660  * @param[in] flow
5661  *   Parent flow structure pointer.
5662  * @param[in] attr
5663  *   Flow rule attributes.
5664  * @param[in] items
5665  *   Pattern specification (list terminated by the END pattern item).
5666  * @param[in] actions
5667  *   Associated actions (list terminated by the END action).
5668  * @param[in] flow_split_info
5669  *   Pointer to flow split info structure.
5670  * @param[out] error
5671  *   Perform verbose error reporting if not NULL.
5672  * @return
5673  *   0 on success, negative value otherwise
5674  */
5675 static int
5676 flow_create_split_meter(struct rte_eth_dev *dev,
5677                         struct rte_flow *flow,
5678                         const struct rte_flow_attr *attr,
5679                         const struct rte_flow_item items[],
5680                         const struct rte_flow_action actions[],
5681                         struct mlx5_flow_split_info *flow_split_info,
5682                         struct rte_flow_error *error)
5683 {
5684         struct mlx5_priv *priv = dev->data->dev_private;
5685         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5686         struct rte_flow_action *sfx_actions = NULL;
5687         struct rte_flow_action *pre_actions = NULL;
5688         struct rte_flow_item *sfx_items = NULL;
5689         struct mlx5_flow *dev_flow = NULL;
5690         struct rte_flow_attr sfx_attr = *attr;
5691         struct mlx5_flow_meter_info *fm = NULL;
5692         uint8_t skip_scale_restore;
5693         bool has_mtr = false;
5694         bool has_modify = false;
5695         bool set_mtr_reg = true;
5696         uint32_t meter_id = 0;
5697         uint32_t mtr_idx = 0;
5698         uint32_t mtr_flow_id = 0;
5699         size_t act_size;
5700         size_t item_size;
5701         int actions_n = 0;
5702         int ret = 0;
5703
5704         if (priv->mtr_en)
5705                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5706                                                     &has_modify, &meter_id);
5707         if (has_mtr) {
5708                 if (flow->meter) {
5709                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5710                         if (!fm)
5711                                 return rte_flow_error_set(error, EINVAL,
5712                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5713                                                 NULL, "Meter not found.");
5714                 } else {
5715                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5716                         if (!fm)
5717                                 return rte_flow_error_set(error, EINVAL,
5718                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5719                                                 NULL, "Meter not found.");
5720                         ret = mlx5_flow_meter_attach(priv, fm,
5721                                                      &sfx_attr, error);
5722                         if (ret)
5723                                 return -rte_errno;
5724                         flow->meter = mtr_idx;
5725                 }
5726                 MLX5_ASSERT(wks);
5727                 wks->fm = fm;
5728                 /*
5729                  * If it isn't default-policy Meter, and
5730                  * 1. There's no action in flow to change
5731                  *    packet (modify/encap/decap etc.), OR
5732                  * 2. No drop count needed for this meter.
5733                  * no need to use regC to save meter id anymore.
5734                  */
5735                 if (!fm->def_policy && (!has_modify || !fm->drop_cnt))
5736                         set_mtr_reg = false;
5737                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5738                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5739                            sizeof(struct mlx5_rte_flow_action_set_tag);
5740                 /* Suffix items: tag, vlan, port id, end. */
5741 #define METER_SUFFIX_ITEM 4
5742                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5743                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5744                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5745                                           0, SOCKET_ID_ANY);
5746                 if (!sfx_actions)
5747                         return rte_flow_error_set(error, ENOMEM,
5748                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5749                                                   NULL, "no memory to split "
5750                                                   "meter flow");
5751                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5752                              act_size);
5753                 /* There's no suffix flow for meter of non-default policy. */
5754                 if (!fm->def_policy)
5755                         pre_actions = sfx_actions + 1;
5756                 else
5757                         pre_actions = sfx_actions + actions_n;
5758                 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr,
5759                                             items, sfx_items, actions,
5760                                             sfx_actions, pre_actions,
5761                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5762                                             error);
5763                 if (ret) {
5764                         ret = -rte_errno;
5765                         goto exit;
5766                 }
5767                 /* Add the prefix subflow. */
5768                 flow_split_info->prefix_mark = 0;
5769                 skip_scale_restore = flow_split_info->skip_scale;
5770                 flow_split_info->skip_scale |=
5771                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5772                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5773                                               attr, items, pre_actions,
5774                                               flow_split_info, error);
5775                 flow_split_info->skip_scale = skip_scale_restore;
5776                 if (ret) {
5777                         if (mtr_flow_id)
5778                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5779                         ret = -rte_errno;
5780                         goto exit;
5781                 }
5782                 if (mtr_flow_id) {
5783                         dev_flow->handle->split_flow_id = mtr_flow_id;
5784                         dev_flow->handle->is_meter_flow_id = 1;
5785                 }
5786                 if (!fm->def_policy) {
5787                         if (!set_mtr_reg && fm->drop_cnt)
5788                                 ret =
5789                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5790                                                         &sfx_attr, items,
5791                                                         flow_split_info,
5792                                                         fm, error);
5793                         goto exit;
5794                 }
5795                 /* Setting the sfx group atrr. */
5796                 sfx_attr.group = sfx_attr.transfer ?
5797                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5798                                  MLX5_FLOW_TABLE_LEVEL_METER;
5799                 flow_split_info->prefix_layers =
5800                                 flow_get_prefix_layer_flags(dev_flow);
5801                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5802                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5803         }
5804         /* Add the prefix subflow. */
5805         ret = flow_create_split_metadata(dev, flow,
5806                                          &sfx_attr, sfx_items ?
5807                                          sfx_items : items,
5808                                          sfx_actions ? sfx_actions : actions,
5809                                          flow_split_info, error);
5810 exit:
5811         if (sfx_actions)
5812                 mlx5_free(sfx_actions);
5813         return ret;
5814 }
5815
5816 /**
5817  * The splitting for sample feature.
5818  *
5819  * Once Sample action is detected in the action list, the flow actions should
5820  * be split into prefix sub flow and suffix sub flow.
5821  *
5822  * The original items remain in the prefix sub flow, all actions preceding the
5823  * sample action and the sample action itself will be copied to the prefix
5824  * sub flow, the actions following the sample action will be copied to the
5825  * suffix sub flow, Queue action always be located in the suffix sub flow.
5826  *
5827  * In order to make the packet from prefix sub flow matches with suffix sub
5828  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
5829  * flow uses tag item with the unique flow id.
5830  *
5831  * @param dev
5832  *   Pointer to Ethernet device.
5833  * @param[in] flow
5834  *   Parent flow structure pointer.
5835  * @param[in] attr
5836  *   Flow rule attributes.
5837  * @param[in] items
5838  *   Pattern specification (list terminated by the END pattern item).
5839  * @param[in] actions
5840  *   Associated actions (list terminated by the END action).
5841  * @param[in] flow_split_info
5842  *   Pointer to flow split info structure.
5843  * @param[out] error
5844  *   Perform verbose error reporting if not NULL.
5845  * @return
5846  *   0 on success, negative value otherwise
5847  */
5848 static int
5849 flow_create_split_sample(struct rte_eth_dev *dev,
5850                          struct rte_flow *flow,
5851                          const struct rte_flow_attr *attr,
5852                          const struct rte_flow_item items[],
5853                          const struct rte_flow_action actions[],
5854                          struct mlx5_flow_split_info *flow_split_info,
5855                          struct rte_flow_error *error)
5856 {
5857         struct mlx5_priv *priv = dev->data->dev_private;
5858         struct rte_flow_action *sfx_actions = NULL;
5859         struct rte_flow_action *pre_actions = NULL;
5860         struct rte_flow_item *sfx_items = NULL;
5861         struct mlx5_flow *dev_flow = NULL;
5862         struct rte_flow_attr sfx_attr = *attr;
5863 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5864         struct mlx5_flow_dv_sample_resource *sample_res;
5865         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
5866         struct mlx5_flow_tbl_resource *sfx_tbl;
5867 #endif
5868         size_t act_size;
5869         size_t item_size;
5870         uint32_t fdb_tx = 0;
5871         int32_t tag_id = 0;
5872         int actions_n = 0;
5873         int sample_action_pos;
5874         int qrss_action_pos;
5875         int add_tag = 0;
5876         int modify_after_mirror = 0;
5877         uint16_t jump_table = 0;
5878         const uint32_t next_ft_step = 1;
5879         int ret = 0;
5880
5881         if (priv->sampler_en)
5882                 actions_n = flow_check_match_action(actions, attr,
5883                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
5884                                         &sample_action_pos, &qrss_action_pos,
5885                                         &modify_after_mirror);
5886         if (actions_n) {
5887                 /* The prefix actions must includes sample, tag, end. */
5888                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
5889                            + sizeof(struct mlx5_rte_flow_action_set_tag);
5890                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
5891                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5892                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
5893                                           item_size), 0, SOCKET_ID_ANY);
5894                 if (!sfx_actions)
5895                         return rte_flow_error_set(error, ENOMEM,
5896                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5897                                                   NULL, "no memory to split "
5898                                                   "sample flow");
5899                 /* The representor_id is -1 for uplink. */
5900                 fdb_tx = (attr->transfer && priv->representor_id != -1);
5901                 /*
5902                  * When reg_c_preserve is set, metadata registers Cx preserve
5903                  * their value even through packet duplication.
5904                  */
5905                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
5906                 if (add_tag)
5907                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
5908                                         + act_size);
5909                 if (modify_after_mirror)
5910                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
5911                                      next_ft_step;
5912                 pre_actions = sfx_actions + actions_n;
5913                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
5914                                                 actions, sfx_actions,
5915                                                 pre_actions, actions_n,
5916                                                 sample_action_pos,
5917                                                 qrss_action_pos, jump_table,
5918                                                 error);
5919                 if (tag_id < 0 || (add_tag && !tag_id)) {
5920                         ret = -rte_errno;
5921                         goto exit;
5922                 }
5923                 if (modify_after_mirror)
5924                         flow_split_info->skip_scale =
5925                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5926                 /* Add the prefix subflow. */
5927                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5928                                               items, pre_actions,
5929                                               flow_split_info, error);
5930                 if (ret) {
5931                         ret = -rte_errno;
5932                         goto exit;
5933                 }
5934                 dev_flow->handle->split_flow_id = tag_id;
5935 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5936                 if (!modify_after_mirror) {
5937                         /* Set the sfx group attr. */
5938                         sample_res = (struct mlx5_flow_dv_sample_resource *)
5939                                                 dev_flow->dv.sample_res;
5940                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
5941                                                 sample_res->normal_path_tbl;
5942                         sfx_tbl_data = container_of(sfx_tbl,
5943                                                 struct mlx5_flow_tbl_data_entry,
5944                                                 tbl);
5945                         sfx_attr.group = sfx_attr.transfer ?
5946                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
5947                 } else {
5948                         MLX5_ASSERT(attr->transfer);
5949                         sfx_attr.group = jump_table;
5950                 }
5951                 flow_split_info->prefix_layers =
5952                                 flow_get_prefix_layer_flags(dev_flow);
5953                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5954                 /* Suffix group level already be scaled with factor, set
5955                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
5956                  * again in translation.
5957                  */
5958                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5959 #endif
5960         }
5961         /* Add the suffix subflow. */
5962         ret = flow_create_split_meter(dev, flow, &sfx_attr,
5963                                       sfx_items ? sfx_items : items,
5964                                       sfx_actions ? sfx_actions : actions,
5965                                       flow_split_info, error);
5966 exit:
5967         if (sfx_actions)
5968                 mlx5_free(sfx_actions);
5969         return ret;
5970 }
5971
5972 /**
5973  * Split the flow to subflow set. The splitters might be linked
5974  * in the chain, like this:
5975  * flow_create_split_outer() calls:
5976  *   flow_create_split_meter() calls:
5977  *     flow_create_split_metadata(meter_subflow_0) calls:
5978  *       flow_create_split_inner(metadata_subflow_0)
5979  *       flow_create_split_inner(metadata_subflow_1)
5980  *       flow_create_split_inner(metadata_subflow_2)
5981  *     flow_create_split_metadata(meter_subflow_1) calls:
5982  *       flow_create_split_inner(metadata_subflow_0)
5983  *       flow_create_split_inner(metadata_subflow_1)
5984  *       flow_create_split_inner(metadata_subflow_2)
5985  *
5986  * This provide flexible way to add new levels of flow splitting.
5987  * The all of successfully created subflows are included to the
5988  * parent flow dev_flow list.
5989  *
5990  * @param dev
5991  *   Pointer to Ethernet device.
5992  * @param[in] flow
5993  *   Parent flow structure pointer.
5994  * @param[in] attr
5995  *   Flow rule attributes.
5996  * @param[in] items
5997  *   Pattern specification (list terminated by the END pattern item).
5998  * @param[in] actions
5999  *   Associated actions (list terminated by the END action).
6000  * @param[in] flow_split_info
6001  *   Pointer to flow split info structure.
6002  * @param[out] error
6003  *   Perform verbose error reporting if not NULL.
6004  * @return
6005  *   0 on success, negative value otherwise
6006  */
6007 static int
6008 flow_create_split_outer(struct rte_eth_dev *dev,
6009                         struct rte_flow *flow,
6010                         const struct rte_flow_attr *attr,
6011                         const struct rte_flow_item items[],
6012                         const struct rte_flow_action actions[],
6013                         struct mlx5_flow_split_info *flow_split_info,
6014                         struct rte_flow_error *error)
6015 {
6016         int ret;
6017
6018         ret = flow_create_split_sample(dev, flow, attr, items,
6019                                        actions, flow_split_info, error);
6020         MLX5_ASSERT(ret <= 0);
6021         return ret;
6022 }
6023
6024 static inline struct mlx5_flow_tunnel *
6025 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6026 {
6027         struct mlx5_flow_tunnel *tunnel;
6028
6029 #pragma GCC diagnostic push
6030 #pragma GCC diagnostic ignored "-Wcast-qual"
6031         tunnel = (typeof(tunnel))flow->tunnel;
6032 #pragma GCC diagnostic pop
6033
6034         return tunnel;
6035 }
6036
6037 /**
6038  * Adjust flow RSS workspace if needed.
6039  *
6040  * @param wks
6041  *   Pointer to thread flow work space.
6042  * @param rss_desc
6043  *   Pointer to RSS descriptor.
6044  * @param[in] nrssq_num
6045  *   New RSS queue number.
6046  *
6047  * @return
6048  *   0 on success, -1 otherwise and rte_errno is set.
6049  */
6050 static int
6051 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6052                           struct mlx5_flow_rss_desc *rss_desc,
6053                           uint32_t nrssq_num)
6054 {
6055         if (likely(nrssq_num <= wks->rssq_num))
6056                 return 0;
6057         rss_desc->queue = realloc(rss_desc->queue,
6058                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6059         if (!rss_desc->queue) {
6060                 rte_errno = ENOMEM;
6061                 return -1;
6062         }
6063         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6064         return 0;
6065 }
6066
6067 /**
6068  * Create a flow and add it to @p list.
6069  *
6070  * @param dev
6071  *   Pointer to Ethernet device.
6072  * @param list
6073  *   Pointer to a TAILQ flow list. If this parameter NULL,
6074  *   no list insertion occurred, flow is just created,
6075  *   this is caller's responsibility to track the
6076  *   created flow.
6077  * @param[in] attr
6078  *   Flow rule attributes.
6079  * @param[in] items
6080  *   Pattern specification (list terminated by the END pattern item).
6081  * @param[in] actions
6082  *   Associated actions (list terminated by the END action).
6083  * @param[in] external
6084  *   This flow rule is created by request external to PMD.
6085  * @param[out] error
6086  *   Perform verbose error reporting if not NULL.
6087  *
6088  * @return
6089  *   A flow index on success, 0 otherwise and rte_errno is set.
6090  */
6091 static uint32_t
6092 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
6093                  const struct rte_flow_attr *attr,
6094                  const struct rte_flow_item items[],
6095                  const struct rte_flow_action original_actions[],
6096                  bool external, struct rte_flow_error *error)
6097 {
6098         struct mlx5_priv *priv = dev->data->dev_private;
6099         struct rte_flow *flow = NULL;
6100         struct mlx5_flow *dev_flow;
6101         const struct rte_flow_action_rss *rss = NULL;
6102         struct mlx5_translated_action_handle
6103                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6104         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6105         union {
6106                 struct mlx5_flow_expand_rss buf;
6107                 uint8_t buffer[2048];
6108         } expand_buffer;
6109         union {
6110                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6111                 uint8_t buffer[2048];
6112         } actions_rx;
6113         union {
6114                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6115                 uint8_t buffer[2048];
6116         } actions_hairpin_tx;
6117         union {
6118                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6119                 uint8_t buffer[2048];
6120         } items_tx;
6121         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6122         struct mlx5_flow_rss_desc *rss_desc;
6123         const struct rte_flow_action *p_actions_rx;
6124         uint32_t i;
6125         uint32_t idx = 0;
6126         int hairpin_flow;
6127         struct rte_flow_attr attr_tx = { .priority = 0 };
6128         const struct rte_flow_action *actions;
6129         struct rte_flow_action *translated_actions = NULL;
6130         struct mlx5_flow_tunnel *tunnel;
6131         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6132         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6133         struct mlx5_flow_split_info flow_split_info = {
6134                 .external = !!external,
6135                 .skip_scale = 0,
6136                 .flow_idx = 0,
6137                 .prefix_mark = 0,
6138                 .prefix_layers = 0,
6139                 .table_id = 0
6140         };
6141         int ret;
6142
6143         MLX5_ASSERT(wks);
6144         rss_desc = &wks->rss_desc;
6145         ret = flow_action_handles_translate(dev, original_actions,
6146                                             indir_actions,
6147                                             &indir_actions_n,
6148                                             &translated_actions, error);
6149         if (ret < 0) {
6150                 MLX5_ASSERT(translated_actions == NULL);
6151                 return 0;
6152         }
6153         actions = translated_actions ? translated_actions : original_actions;
6154         p_actions_rx = actions;
6155         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6156         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6157                                 external, hairpin_flow, error);
6158         if (ret < 0)
6159                 goto error_before_hairpin_split;
6160         flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
6161         if (!flow) {
6162                 rte_errno = ENOMEM;
6163                 goto error_before_hairpin_split;
6164         }
6165         if (hairpin_flow > 0) {
6166                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6167                         rte_errno = EINVAL;
6168                         goto error_before_hairpin_split;
6169                 }
6170                 flow_hairpin_split(dev, actions, actions_rx.actions,
6171                                    actions_hairpin_tx.actions, items_tx.items,
6172                                    idx);
6173                 p_actions_rx = actions_rx.actions;
6174         }
6175         flow_split_info.flow_idx = idx;
6176         flow->drv_type = flow_get_drv_type(dev, attr);
6177         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6178                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6179         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6180         /* RSS Action only works on NIC RX domain */
6181         if (attr->ingress && !attr->transfer)
6182                 rss = flow_get_rss_action(dev, p_actions_rx);
6183         if (rss) {
6184                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6185                         return 0;
6186                 /*
6187                  * The following information is required by
6188                  * mlx5_flow_hashfields_adjust() in advance.
6189                  */
6190                 rss_desc->level = rss->level;
6191                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6192                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6193         }
6194         flow->dev_handles = 0;
6195         if (rss && rss->types) {
6196                 unsigned int graph_root;
6197
6198                 graph_root = find_graph_root(items, rss->level);
6199                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6200                                            items, rss->types,
6201                                            mlx5_support_expansion, graph_root);
6202                 MLX5_ASSERT(ret > 0 &&
6203                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6204                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6205                         for (i = 0; i < buf->entries; ++i)
6206                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6207                 }
6208         } else {
6209                 buf->entries = 1;
6210                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6211         }
6212         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6213                                                       indir_actions_n);
6214         for (i = 0; i < buf->entries; ++i) {
6215                 /* Initialize flow split data. */
6216                 flow_split_info.prefix_layers = 0;
6217                 flow_split_info.prefix_mark = 0;
6218                 flow_split_info.skip_scale = 0;
6219                 /*
6220                  * The splitter may create multiple dev_flows,
6221                  * depending on configuration. In the simplest
6222                  * case it just creates unmodified original flow.
6223                  */
6224                 ret = flow_create_split_outer(dev, flow, attr,
6225                                               buf->entry[i].pattern,
6226                                               p_actions_rx, &flow_split_info,
6227                                               error);
6228                 if (ret < 0)
6229                         goto error;
6230                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6231                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6232                                                            p_actions_rx,
6233                                                            idx,
6234                                                            wks->flows[0].tunnel,
6235                                                            &default_miss_ctx,
6236                                                            error);
6237                         if (ret < 0) {
6238                                 mlx5_free(default_miss_ctx.queue);
6239                                 goto error;
6240                         }
6241                 }
6242         }
6243         /* Create the tx flow. */
6244         if (hairpin_flow) {
6245                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6246                 attr_tx.ingress = 0;
6247                 attr_tx.egress = 1;
6248                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6249                                          actions_hairpin_tx.actions,
6250                                          idx, error);
6251                 if (!dev_flow)
6252                         goto error;
6253                 dev_flow->flow = flow;
6254                 dev_flow->external = 0;
6255                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6256                               dev_flow->handle, next);
6257                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6258                                          items_tx.items,
6259                                          actions_hairpin_tx.actions, error);
6260                 if (ret < 0)
6261                         goto error;
6262         }
6263         /*
6264          * Update the metadata register copy table. If extensive
6265          * metadata feature is enabled and registers are supported
6266          * we might create the extra rte_flow for each unique
6267          * MARK/FLAG action ID.
6268          *
6269          * The table is updated for ingress Flows only, because
6270          * the egress Flows belong to the different device and
6271          * copy table should be updated in peer NIC Rx domain.
6272          */
6273         if (attr->ingress &&
6274             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6275                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6276                 if (ret)
6277                         goto error;
6278         }
6279         /*
6280          * If the flow is external (from application) OR device is started,
6281          * OR mreg discover, then apply immediately.
6282          */
6283         if (external || dev->data->dev_started ||
6284             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6285              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6286                 ret = flow_drv_apply(dev, flow, error);
6287                 if (ret < 0)
6288                         goto error;
6289         }
6290         if (list) {
6291                 rte_spinlock_lock(&priv->flow_list_lock);
6292                 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
6293                              flow, next);
6294                 rte_spinlock_unlock(&priv->flow_list_lock);
6295         }
6296         flow_rxq_flags_set(dev, flow);
6297         rte_free(translated_actions);
6298         tunnel = flow_tunnel_from_rule(wks->flows);
6299         if (tunnel) {
6300                 flow->tunnel = 1;
6301                 flow->tunnel_id = tunnel->tunnel_id;
6302                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6303                 mlx5_free(default_miss_ctx.queue);
6304         }
6305         mlx5_flow_pop_thread_workspace();
6306         return idx;
6307 error:
6308         MLX5_ASSERT(flow);
6309         ret = rte_errno; /* Save rte_errno before cleanup. */
6310         flow_mreg_del_copy_action(dev, flow);
6311         flow_drv_destroy(dev, flow);
6312         if (rss_desc->shared_rss)
6313                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6314                         mlx5_ipool_get
6315                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6316                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6317         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
6318         rte_errno = ret; /* Restore rte_errno. */
6319         ret = rte_errno;
6320         rte_errno = ret;
6321         mlx5_flow_pop_thread_workspace();
6322 error_before_hairpin_split:
6323         rte_free(translated_actions);
6324         return 0;
6325 }
6326
6327 /**
6328  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6329  * incoming packets to table 1.
6330  *
6331  * Other flow rules, requested for group n, will be created in
6332  * e-switch table n+1.
6333  * Jump action to e-switch group n will be created to group n+1.
6334  *
6335  * Used when working in switchdev mode, to utilise advantages of table 1
6336  * and above.
6337  *
6338  * @param dev
6339  *   Pointer to Ethernet device.
6340  *
6341  * @return
6342  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6343  */
6344 struct rte_flow *
6345 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6346 {
6347         const struct rte_flow_attr attr = {
6348                 .group = 0,
6349                 .priority = 0,
6350                 .ingress = 1,
6351                 .egress = 0,
6352                 .transfer = 1,
6353         };
6354         const struct rte_flow_item pattern = {
6355                 .type = RTE_FLOW_ITEM_TYPE_END,
6356         };
6357         struct rte_flow_action_jump jump = {
6358                 .group = 1,
6359         };
6360         const struct rte_flow_action actions[] = {
6361                 {
6362                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6363                         .conf = &jump,
6364                 },
6365                 {
6366                         .type = RTE_FLOW_ACTION_TYPE_END,
6367                 },
6368         };
6369         struct mlx5_priv *priv = dev->data->dev_private;
6370         struct rte_flow_error error;
6371
6372         return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
6373                                                    &attr, &pattern,
6374                                                    actions, false, &error);
6375 }
6376
6377 /**
6378  * Validate a flow supported by the NIC.
6379  *
6380  * @see rte_flow_validate()
6381  * @see rte_flow_ops
6382  */
6383 int
6384 mlx5_flow_validate(struct rte_eth_dev *dev,
6385                    const struct rte_flow_attr *attr,
6386                    const struct rte_flow_item items[],
6387                    const struct rte_flow_action original_actions[],
6388                    struct rte_flow_error *error)
6389 {
6390         int hairpin_flow;
6391         struct mlx5_translated_action_handle
6392                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6393         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6394         const struct rte_flow_action *actions;
6395         struct rte_flow_action *translated_actions = NULL;
6396         int ret = flow_action_handles_translate(dev, original_actions,
6397                                                 indir_actions,
6398                                                 &indir_actions_n,
6399                                                 &translated_actions, error);
6400
6401         if (ret)
6402                 return ret;
6403         actions = translated_actions ? translated_actions : original_actions;
6404         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6405         ret = flow_drv_validate(dev, attr, items, actions,
6406                                 true, hairpin_flow, error);
6407         rte_free(translated_actions);
6408         return ret;
6409 }
6410
6411 /**
6412  * Create a flow.
6413  *
6414  * @see rte_flow_create()
6415  * @see rte_flow_ops
6416  */
6417 struct rte_flow *
6418 mlx5_flow_create(struct rte_eth_dev *dev,
6419                  const struct rte_flow_attr *attr,
6420                  const struct rte_flow_item items[],
6421                  const struct rte_flow_action actions[],
6422                  struct rte_flow_error *error)
6423 {
6424         struct mlx5_priv *priv = dev->data->dev_private;
6425
6426         /*
6427          * If the device is not started yet, it is not allowed to created a
6428          * flow from application. PMD default flows and traffic control flows
6429          * are not affected.
6430          */
6431         if (unlikely(!dev->data->dev_started)) {
6432                 DRV_LOG(DEBUG, "port %u is not started when "
6433                         "inserting a flow", dev->data->port_id);
6434                 rte_flow_error_set(error, ENODEV,
6435                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6436                                    NULL,
6437                                    "port not started");
6438                 return NULL;
6439         }
6440
6441         return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
6442                                   attr, items, actions, true, error);
6443 }
6444
6445 /**
6446  * Destroy a flow in a list.
6447  *
6448  * @param dev
6449  *   Pointer to Ethernet device.
6450  * @param list
6451  *   Pointer to the Indexed flow list. If this parameter NULL,
6452  *   there is no flow removal from the list. Be noted that as
6453  *   flow is add to the indexed list, memory of the indexed
6454  *   list points to maybe changed as flow destroyed.
6455  * @param[in] flow_idx
6456  *   Index of flow to destroy.
6457  */
6458 static void
6459 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
6460                   uint32_t flow_idx)
6461 {
6462         struct mlx5_priv *priv = dev->data->dev_private;
6463         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
6464                                                [MLX5_IPOOL_RTE_FLOW], flow_idx);
6465
6466         if (!flow)
6467                 return;
6468         /*
6469          * Update RX queue flags only if port is started, otherwise it is
6470          * already clean.
6471          */
6472         if (dev->data->dev_started)
6473                 flow_rxq_flags_trim(dev, flow);
6474         flow_drv_destroy(dev, flow);
6475         if (list) {
6476                 rte_spinlock_lock(&priv->flow_list_lock);
6477                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
6478                              flow_idx, flow, next);
6479                 rte_spinlock_unlock(&priv->flow_list_lock);
6480         }
6481         if (flow->tunnel) {
6482                 struct mlx5_flow_tunnel *tunnel;
6483
6484                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6485                 RTE_VERIFY(tunnel);
6486                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6487                         mlx5_flow_tunnel_free(dev, tunnel);
6488         }
6489         flow_mreg_del_copy_action(dev, flow);
6490         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
6491 }
6492
6493 /**
6494  * Destroy all flows.
6495  *
6496  * @param dev
6497  *   Pointer to Ethernet device.
6498  * @param list
6499  *   Pointer to the Indexed flow list.
6500  * @param active
6501  *   If flushing is called avtively.
6502  */
6503 void
6504 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
6505 {
6506         uint32_t num_flushed = 0;
6507
6508         while (*list) {
6509                 flow_list_destroy(dev, list, *list);
6510                 num_flushed++;
6511         }
6512         if (active) {
6513                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6514                         dev->data->port_id, num_flushed);
6515         }
6516 }
6517
6518 /**
6519  * Stop all default actions for flows.
6520  *
6521  * @param dev
6522  *   Pointer to Ethernet device.
6523  */
6524 void
6525 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6526 {
6527         flow_mreg_del_default_copy_action(dev);
6528         flow_rxq_flags_clear(dev);
6529 }
6530
6531 /**
6532  * Start all default actions for flows.
6533  *
6534  * @param dev
6535  *   Pointer to Ethernet device.
6536  * @return
6537  *   0 on success, a negative errno value otherwise and rte_errno is set.
6538  */
6539 int
6540 mlx5_flow_start_default(struct rte_eth_dev *dev)
6541 {
6542         struct rte_flow_error error;
6543
6544         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6545         return flow_mreg_add_default_copy_action(dev, &error);
6546 }
6547
6548 /**
6549  * Release key of thread specific flow workspace data.
6550  */
6551 void
6552 flow_release_workspace(void *data)
6553 {
6554         struct mlx5_flow_workspace *wks = data;
6555         struct mlx5_flow_workspace *next;
6556
6557         while (wks) {
6558                 next = wks->next;
6559                 free(wks->rss_desc.queue);
6560                 free(wks);
6561                 wks = next;
6562         }
6563 }
6564
6565 /**
6566  * Get thread specific current flow workspace.
6567  *
6568  * @return pointer to thread specific flow workspace data, NULL on error.
6569  */
6570 struct mlx5_flow_workspace*
6571 mlx5_flow_get_thread_workspace(void)
6572 {
6573         struct mlx5_flow_workspace *data;
6574
6575         data = mlx5_flow_os_get_specific_workspace();
6576         MLX5_ASSERT(data && data->inuse);
6577         if (!data || !data->inuse)
6578                 DRV_LOG(ERR, "flow workspace not initialized.");
6579         return data;
6580 }
6581
6582 /**
6583  * Allocate and init new flow workspace.
6584  *
6585  * @return pointer to flow workspace data, NULL on error.
6586  */
6587 static struct mlx5_flow_workspace*
6588 flow_alloc_thread_workspace(void)
6589 {
6590         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6591
6592         if (!data) {
6593                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6594                         "memory.");
6595                 return NULL;
6596         }
6597         data->rss_desc.queue = calloc(1,
6598                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6599         if (!data->rss_desc.queue)
6600                 goto err;
6601         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6602         return data;
6603 err:
6604         if (data->rss_desc.queue)
6605                 free(data->rss_desc.queue);
6606         free(data);
6607         return NULL;
6608 }
6609
6610 /**
6611  * Get new thread specific flow workspace.
6612  *
6613  * If current workspace inuse, create new one and set as current.
6614  *
6615  * @return pointer to thread specific flow workspace data, NULL on error.
6616  */
6617 static struct mlx5_flow_workspace*
6618 mlx5_flow_push_thread_workspace(void)
6619 {
6620         struct mlx5_flow_workspace *curr;
6621         struct mlx5_flow_workspace *data;
6622
6623         curr = mlx5_flow_os_get_specific_workspace();
6624         if (!curr) {
6625                 data = flow_alloc_thread_workspace();
6626                 if (!data)
6627                         return NULL;
6628         } else if (!curr->inuse) {
6629                 data = curr;
6630         } else if (curr->next) {
6631                 data = curr->next;
6632         } else {
6633                 data = flow_alloc_thread_workspace();
6634                 if (!data)
6635                         return NULL;
6636                 curr->next = data;
6637                 data->prev = curr;
6638         }
6639         data->inuse = 1;
6640         data->flow_idx = 0;
6641         /* Set as current workspace */
6642         if (mlx5_flow_os_set_specific_workspace(data))
6643                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6644         return data;
6645 }
6646
6647 /**
6648  * Close current thread specific flow workspace.
6649  *
6650  * If previous workspace available, set it as current.
6651  *
6652  * @return pointer to thread specific flow workspace data, NULL on error.
6653  */
6654 static void
6655 mlx5_flow_pop_thread_workspace(void)
6656 {
6657         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6658
6659         if (!data)
6660                 return;
6661         if (!data->inuse) {
6662                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6663                 return;
6664         }
6665         data->inuse = 0;
6666         if (!data->prev)
6667                 return;
6668         if (mlx5_flow_os_set_specific_workspace(data->prev))
6669                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6670 }
6671
6672 /**
6673  * Verify the flow list is empty
6674  *
6675  * @param dev
6676  *  Pointer to Ethernet device.
6677  *
6678  * @return the number of flows not released.
6679  */
6680 int
6681 mlx5_flow_verify(struct rte_eth_dev *dev)
6682 {
6683         struct mlx5_priv *priv = dev->data->dev_private;
6684         struct rte_flow *flow;
6685         uint32_t idx;
6686         int ret = 0;
6687
6688         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
6689                       flow, next) {
6690                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
6691                         dev->data->port_id, (void *)flow);
6692                 ++ret;
6693         }
6694         return ret;
6695 }
6696
6697 /**
6698  * Enable default hairpin egress flow.
6699  *
6700  * @param dev
6701  *   Pointer to Ethernet device.
6702  * @param queue
6703  *   The queue index.
6704  *
6705  * @return
6706  *   0 on success, a negative errno value otherwise and rte_errno is set.
6707  */
6708 int
6709 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6710                             uint32_t queue)
6711 {
6712         struct mlx5_priv *priv = dev->data->dev_private;
6713         const struct rte_flow_attr attr = {
6714                 .egress = 1,
6715                 .priority = 0,
6716         };
6717         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6718                 .queue = queue,
6719         };
6720         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6721                 .queue = UINT32_MAX,
6722         };
6723         struct rte_flow_item items[] = {
6724                 {
6725                         .type = (enum rte_flow_item_type)
6726                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6727                         .spec = &queue_spec,
6728                         .last = NULL,
6729                         .mask = &queue_mask,
6730                 },
6731                 {
6732                         .type = RTE_FLOW_ITEM_TYPE_END,
6733                 },
6734         };
6735         struct rte_flow_action_jump jump = {
6736                 .group = MLX5_HAIRPIN_TX_TABLE,
6737         };
6738         struct rte_flow_action actions[2];
6739         uint32_t flow_idx;
6740         struct rte_flow_error error;
6741
6742         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6743         actions[0].conf = &jump;
6744         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6745         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6746                                 &attr, items, actions, false, &error);
6747         if (!flow_idx) {
6748                 DRV_LOG(DEBUG,
6749                         "Failed to create ctrl flow: rte_errno(%d),"
6750                         " type(%d), message(%s)",
6751                         rte_errno, error.type,
6752                         error.message ? error.message : " (no stated reason)");
6753                 return -rte_errno;
6754         }
6755         return 0;
6756 }
6757
6758 /**
6759  * Enable a control flow configured from the control plane.
6760  *
6761  * @param dev
6762  *   Pointer to Ethernet device.
6763  * @param eth_spec
6764  *   An Ethernet flow spec to apply.
6765  * @param eth_mask
6766  *   An Ethernet flow mask to apply.
6767  * @param vlan_spec
6768  *   A VLAN flow spec to apply.
6769  * @param vlan_mask
6770  *   A VLAN flow mask to apply.
6771  *
6772  * @return
6773  *   0 on success, a negative errno value otherwise and rte_errno is set.
6774  */
6775 int
6776 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6777                     struct rte_flow_item_eth *eth_spec,
6778                     struct rte_flow_item_eth *eth_mask,
6779                     struct rte_flow_item_vlan *vlan_spec,
6780                     struct rte_flow_item_vlan *vlan_mask)
6781 {
6782         struct mlx5_priv *priv = dev->data->dev_private;
6783         const struct rte_flow_attr attr = {
6784                 .ingress = 1,
6785                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6786         };
6787         struct rte_flow_item items[] = {
6788                 {
6789                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6790                         .spec = eth_spec,
6791                         .last = NULL,
6792                         .mask = eth_mask,
6793                 },
6794                 {
6795                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6796                                               RTE_FLOW_ITEM_TYPE_END,
6797                         .spec = vlan_spec,
6798                         .last = NULL,
6799                         .mask = vlan_mask,
6800                 },
6801                 {
6802                         .type = RTE_FLOW_ITEM_TYPE_END,
6803                 },
6804         };
6805         uint16_t queue[priv->reta_idx_n];
6806         struct rte_flow_action_rss action_rss = {
6807                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6808                 .level = 0,
6809                 .types = priv->rss_conf.rss_hf,
6810                 .key_len = priv->rss_conf.rss_key_len,
6811                 .queue_num = priv->reta_idx_n,
6812                 .key = priv->rss_conf.rss_key,
6813                 .queue = queue,
6814         };
6815         struct rte_flow_action actions[] = {
6816                 {
6817                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6818                         .conf = &action_rss,
6819                 },
6820                 {
6821                         .type = RTE_FLOW_ACTION_TYPE_END,
6822                 },
6823         };
6824         uint32_t flow_idx;
6825         struct rte_flow_error error;
6826         unsigned int i;
6827
6828         if (!priv->reta_idx_n || !priv->rxqs_n) {
6829                 return 0;
6830         }
6831         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6832                 action_rss.types = 0;
6833         for (i = 0; i != priv->reta_idx_n; ++i)
6834                 queue[i] = (*priv->reta_idx)[i];
6835         flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6836                                 &attr, items, actions, false, &error);
6837         if (!flow_idx)
6838                 return -rte_errno;
6839         return 0;
6840 }
6841
6842 /**
6843  * Enable a flow control configured from the control plane.
6844  *
6845  * @param dev
6846  *   Pointer to Ethernet device.
6847  * @param eth_spec
6848  *   An Ethernet flow spec to apply.
6849  * @param eth_mask
6850  *   An Ethernet flow mask to apply.
6851  *
6852  * @return
6853  *   0 on success, a negative errno value otherwise and rte_errno is set.
6854  */
6855 int
6856 mlx5_ctrl_flow(struct rte_eth_dev *dev,
6857                struct rte_flow_item_eth *eth_spec,
6858                struct rte_flow_item_eth *eth_mask)
6859 {
6860         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
6861 }
6862
6863 /**
6864  * Create default miss flow rule matching lacp traffic
6865  *
6866  * @param dev
6867  *   Pointer to Ethernet device.
6868  * @param eth_spec
6869  *   An Ethernet flow spec to apply.
6870  *
6871  * @return
6872  *   0 on success, a negative errno value otherwise and rte_errno is set.
6873  */
6874 int
6875 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
6876 {
6877         struct mlx5_priv *priv = dev->data->dev_private;
6878         /*
6879          * The LACP matching is done by only using ether type since using
6880          * a multicast dst mac causes kernel to give low priority to this flow.
6881          */
6882         static const struct rte_flow_item_eth lacp_spec = {
6883                 .type = RTE_BE16(0x8809),
6884         };
6885         static const struct rte_flow_item_eth lacp_mask = {
6886                 .type = 0xffff,
6887         };
6888         const struct rte_flow_attr attr = {
6889                 .ingress = 1,
6890         };
6891         struct rte_flow_item items[] = {
6892                 {
6893                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6894                         .spec = &lacp_spec,
6895                         .mask = &lacp_mask,
6896                 },
6897                 {
6898                         .type = RTE_FLOW_ITEM_TYPE_END,
6899                 },
6900         };
6901         struct rte_flow_action actions[] = {
6902                 {
6903                         .type = (enum rte_flow_action_type)
6904                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
6905                 },
6906                 {
6907                         .type = RTE_FLOW_ACTION_TYPE_END,
6908                 },
6909         };
6910         struct rte_flow_error error;
6911         uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
6912                                 &attr, items, actions, false, &error);
6913
6914         if (!flow_idx)
6915                 return -rte_errno;
6916         return 0;
6917 }
6918
6919 /**
6920  * Destroy a flow.
6921  *
6922  * @see rte_flow_destroy()
6923  * @see rte_flow_ops
6924  */
6925 int
6926 mlx5_flow_destroy(struct rte_eth_dev *dev,
6927                   struct rte_flow *flow,
6928                   struct rte_flow_error *error __rte_unused)
6929 {
6930         struct mlx5_priv *priv = dev->data->dev_private;
6931
6932         flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
6933         return 0;
6934 }
6935
6936 /**
6937  * Destroy all flows.
6938  *
6939  * @see rte_flow_flush()
6940  * @see rte_flow_ops
6941  */
6942 int
6943 mlx5_flow_flush(struct rte_eth_dev *dev,
6944                 struct rte_flow_error *error __rte_unused)
6945 {
6946         struct mlx5_priv *priv = dev->data->dev_private;
6947
6948         mlx5_flow_list_flush(dev, &priv->flows, false);
6949         return 0;
6950 }
6951
6952 /**
6953  * Isolated mode.
6954  *
6955  * @see rte_flow_isolate()
6956  * @see rte_flow_ops
6957  */
6958 int
6959 mlx5_flow_isolate(struct rte_eth_dev *dev,
6960                   int enable,
6961                   struct rte_flow_error *error)
6962 {
6963         struct mlx5_priv *priv = dev->data->dev_private;
6964
6965         if (dev->data->dev_started) {
6966                 rte_flow_error_set(error, EBUSY,
6967                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6968                                    NULL,
6969                                    "port must be stopped first");
6970                 return -rte_errno;
6971         }
6972         priv->isolated = !!enable;
6973         if (enable)
6974                 dev->dev_ops = &mlx5_dev_ops_isolate;
6975         else
6976                 dev->dev_ops = &mlx5_dev_ops;
6977
6978         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
6979         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
6980
6981         return 0;
6982 }
6983
6984 /**
6985  * Query a flow.
6986  *
6987  * @see rte_flow_query()
6988  * @see rte_flow_ops
6989  */
6990 static int
6991 flow_drv_query(struct rte_eth_dev *dev,
6992                uint32_t flow_idx,
6993                const struct rte_flow_action *actions,
6994                void *data,
6995                struct rte_flow_error *error)
6996 {
6997         struct mlx5_priv *priv = dev->data->dev_private;
6998         const struct mlx5_flow_driver_ops *fops;
6999         struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
7000                                                [MLX5_IPOOL_RTE_FLOW],
7001                                                flow_idx);
7002         enum mlx5_flow_drv_type ftype;
7003
7004         if (!flow) {
7005                 return rte_flow_error_set(error, ENOENT,
7006                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7007                           NULL,
7008                           "invalid flow handle");
7009         }
7010         ftype = flow->drv_type;
7011         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7012         fops = flow_get_drv_ops(ftype);
7013
7014         return fops->query(dev, flow, actions, data, error);
7015 }
7016
7017 /**
7018  * Query a flow.
7019  *
7020  * @see rte_flow_query()
7021  * @see rte_flow_ops
7022  */
7023 int
7024 mlx5_flow_query(struct rte_eth_dev *dev,
7025                 struct rte_flow *flow,
7026                 const struct rte_flow_action *actions,
7027                 void *data,
7028                 struct rte_flow_error *error)
7029 {
7030         int ret;
7031
7032         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7033                              error);
7034         if (ret < 0)
7035                 return ret;
7036         return 0;
7037 }
7038
7039 /**
7040  * Get rte_flow callbacks.
7041  *
7042  * @param dev
7043  *   Pointer to Ethernet device structure.
7044  * @param ops
7045  *   Pointer to operation-specific structure.
7046  *
7047  * @return 0
7048  */
7049 int
7050 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7051                   const struct rte_flow_ops **ops)
7052 {
7053         *ops = &mlx5_flow_ops;
7054         return 0;
7055 }
7056
7057 /**
7058  * Validate meter policy actions.
7059  * Dispatcher for action type specific validation.
7060  *
7061  * @param[in] dev
7062  *   Pointer to the Ethernet device structure.
7063  * @param[in] action
7064  *   The meter policy action object to validate.
7065  * @param[in] attr
7066  *   Attributes of flow to determine steering domain.
7067  * @param[out] is_rss
7068  *   Is RSS or not.
7069  * @param[out] domain_bitmap
7070  *   Domain bitmap.
7071  * @param[out] is_def_policy
7072  *   Is default policy or not.
7073  * @param[out] error
7074  *   Perform verbose error reporting if not NULL. Initialized in case of
7075  *   error only.
7076  *
7077  * @return
7078  *   0 on success, otherwise negative errno value.
7079  */
7080 int
7081 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7082                         const struct rte_flow_action *actions[RTE_COLORS],
7083                         struct rte_flow_attr *attr,
7084                         bool *is_rss,
7085                         uint8_t *domain_bitmap,
7086                         bool *is_def_policy,
7087                         struct rte_mtr_error *error)
7088 {
7089         const struct mlx5_flow_driver_ops *fops;
7090
7091         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7092         return fops->validate_mtr_acts(dev, actions, attr,
7093                         is_rss, domain_bitmap, is_def_policy, error);
7094 }
7095
7096 /**
7097  * Destroy the meter table set.
7098  *
7099  * @param[in] dev
7100  *   Pointer to Ethernet device.
7101  * @param[in] mtr_policy
7102  *   Meter policy struct.
7103  */
7104 void
7105 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7106                       struct mlx5_flow_meter_policy *mtr_policy)
7107 {
7108         const struct mlx5_flow_driver_ops *fops;
7109
7110         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7111         fops->destroy_mtr_acts(dev, mtr_policy);
7112 }
7113
7114 /**
7115  * Create policy action, lock free,
7116  * (mutex should be acquired by caller).
7117  * Dispatcher for action type specific call.
7118  *
7119  * @param[in] dev
7120  *   Pointer to the Ethernet device structure.
7121  * @param[in] mtr_policy
7122  *   Meter policy struct.
7123  * @param[in] action
7124  *   Action specification used to create meter actions.
7125  * @param[out] error
7126  *   Perform verbose error reporting if not NULL. Initialized in case of
7127  *   error only.
7128  *
7129  * @return
7130  *   0 on success, otherwise negative errno value.
7131  */
7132 int
7133 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7134                       struct mlx5_flow_meter_policy *mtr_policy,
7135                       const struct rte_flow_action *actions[RTE_COLORS],
7136                       struct rte_mtr_error *error)
7137 {
7138         const struct mlx5_flow_driver_ops *fops;
7139
7140         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7141         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7142 }
7143
7144 /**
7145  * Create policy rules, lock free,
7146  * (mutex should be acquired by caller).
7147  * Dispatcher for action type specific call.
7148  *
7149  * @param[in] dev
7150  *   Pointer to the Ethernet device structure.
7151  * @param[in] mtr_policy
7152  *   Meter policy struct.
7153  *
7154  * @return
7155  *   0 on success, -1 otherwise.
7156  */
7157 int
7158 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7159                              struct mlx5_flow_meter_policy *mtr_policy)
7160 {
7161         const struct mlx5_flow_driver_ops *fops;
7162
7163         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7164         return fops->create_policy_rules(dev, mtr_policy);
7165 }
7166
7167 /**
7168  * Destroy policy rules, lock free,
7169  * (mutex should be acquired by caller).
7170  * Dispatcher for action type specific call.
7171  *
7172  * @param[in] dev
7173  *   Pointer to the Ethernet device structure.
7174  * @param[in] mtr_policy
7175  *   Meter policy struct.
7176  */
7177 void
7178 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7179                              struct mlx5_flow_meter_policy *mtr_policy)
7180 {
7181         const struct mlx5_flow_driver_ops *fops;
7182
7183         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7184         fops->destroy_policy_rules(dev, mtr_policy);
7185 }
7186
7187 /**
7188  * Destroy the default policy table set.
7189  *
7190  * @param[in] dev
7191  *   Pointer to Ethernet device.
7192  */
7193 void
7194 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7195 {
7196         const struct mlx5_flow_driver_ops *fops;
7197
7198         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7199         fops->destroy_def_policy(dev);
7200 }
7201
7202 /**
7203  * Destroy the default policy table set.
7204  *
7205  * @param[in] dev
7206  *   Pointer to Ethernet device.
7207  *
7208  * @return
7209  *   0 on success, -1 otherwise.
7210  */
7211 int
7212 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7213 {
7214         const struct mlx5_flow_driver_ops *fops;
7215
7216         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7217         return fops->create_def_policy(dev);
7218 }
7219
7220 /**
7221  * Create the needed meter and suffix tables.
7222  *
7223  * @param[in] dev
7224  *   Pointer to Ethernet device.
7225  *
7226  * @return
7227  *   0 on success, -1 otherwise.
7228  */
7229 int
7230 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7231                         struct mlx5_flow_meter_info *fm,
7232                         uint32_t mtr_idx,
7233                         uint8_t domain_bitmap)
7234 {
7235         const struct mlx5_flow_driver_ops *fops;
7236
7237         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7238         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7239 }
7240
7241 /**
7242  * Destroy the meter table set.
7243  *
7244  * @param[in] dev
7245  *   Pointer to Ethernet device.
7246  * @param[in] tbl
7247  *   Pointer to the meter table set.
7248  */
7249 void
7250 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7251                            struct mlx5_flow_meter_info *fm)
7252 {
7253         const struct mlx5_flow_driver_ops *fops;
7254
7255         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7256         fops->destroy_mtr_tbls(dev, fm);
7257 }
7258
7259 /**
7260  * Destroy the global meter drop table.
7261  *
7262  * @param[in] dev
7263  *   Pointer to Ethernet device.
7264  */
7265 void
7266 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7267 {
7268         const struct mlx5_flow_driver_ops *fops;
7269
7270         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7271         fops->destroy_mtr_drop_tbls(dev);
7272 }
7273
7274 /**
7275  * Destroy the sub policy table with RX queue.
7276  *
7277  * @param[in] dev
7278  *   Pointer to Ethernet device.
7279  * @param[in] mtr_policy
7280  *   Pointer to meter policy table.
7281  */
7282 void
7283 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
7284                 struct mlx5_flow_meter_policy *mtr_policy)
7285 {
7286         const struct mlx5_flow_driver_ops *fops;
7287
7288         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7289         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
7290 }
7291
7292 /**
7293  * Allocate the needed aso flow meter id.
7294  *
7295  * @param[in] dev
7296  *   Pointer to Ethernet device.
7297  *
7298  * @return
7299  *   Index to aso flow meter on success, NULL otherwise.
7300  */
7301 uint32_t
7302 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7303 {
7304         const struct mlx5_flow_driver_ops *fops;
7305
7306         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7307         return fops->create_meter(dev);
7308 }
7309
7310 /**
7311  * Free the aso flow meter id.
7312  *
7313  * @param[in] dev
7314  *   Pointer to Ethernet device.
7315  * @param[in] mtr_idx
7316  *  Index to aso flow meter to be free.
7317  *
7318  * @return
7319  *   0 on success.
7320  */
7321 void
7322 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7323 {
7324         const struct mlx5_flow_driver_ops *fops;
7325
7326         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7327         fops->free_meter(dev, mtr_idx);
7328 }
7329
7330 /**
7331  * Allocate a counter.
7332  *
7333  * @param[in] dev
7334  *   Pointer to Ethernet device structure.
7335  *
7336  * @return
7337  *   Index to allocated counter  on success, 0 otherwise.
7338  */
7339 uint32_t
7340 mlx5_counter_alloc(struct rte_eth_dev *dev)
7341 {
7342         const struct mlx5_flow_driver_ops *fops;
7343         struct rte_flow_attr attr = { .transfer = 0 };
7344
7345         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7346                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7347                 return fops->counter_alloc(dev);
7348         }
7349         DRV_LOG(ERR,
7350                 "port %u counter allocate is not supported.",
7351                  dev->data->port_id);
7352         return 0;
7353 }
7354
7355 /**
7356  * Free a counter.
7357  *
7358  * @param[in] dev
7359  *   Pointer to Ethernet device structure.
7360  * @param[in] cnt
7361  *   Index to counter to be free.
7362  */
7363 void
7364 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7365 {
7366         const struct mlx5_flow_driver_ops *fops;
7367         struct rte_flow_attr attr = { .transfer = 0 };
7368
7369         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7370                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7371                 fops->counter_free(dev, cnt);
7372                 return;
7373         }
7374         DRV_LOG(ERR,
7375                 "port %u counter free is not supported.",
7376                  dev->data->port_id);
7377 }
7378
7379 /**
7380  * Query counter statistics.
7381  *
7382  * @param[in] dev
7383  *   Pointer to Ethernet device structure.
7384  * @param[in] cnt
7385  *   Index to counter to query.
7386  * @param[in] clear
7387  *   Set to clear counter statistics.
7388  * @param[out] pkts
7389  *   The counter hits packets number to save.
7390  * @param[out] bytes
7391  *   The counter hits bytes number to save.
7392  *
7393  * @return
7394  *   0 on success, a negative errno value otherwise.
7395  */
7396 int
7397 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7398                    bool clear, uint64_t *pkts, uint64_t *bytes)
7399 {
7400         const struct mlx5_flow_driver_ops *fops;
7401         struct rte_flow_attr attr = { .transfer = 0 };
7402
7403         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7404                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7405                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7406         }
7407         DRV_LOG(ERR,
7408                 "port %u counter query is not supported.",
7409                  dev->data->port_id);
7410         return -ENOTSUP;
7411 }
7412
7413 /**
7414  * Allocate a new memory for the counter values wrapped by all the needed
7415  * management.
7416  *
7417  * @param[in] sh
7418  *   Pointer to mlx5_dev_ctx_shared object.
7419  *
7420  * @return
7421  *   0 on success, a negative errno value otherwise.
7422  */
7423 static int
7424 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7425 {
7426         struct mlx5_devx_mkey_attr mkey_attr;
7427         struct mlx5_counter_stats_mem_mng *mem_mng;
7428         volatile struct flow_counter_stats *raw_data;
7429         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7430         int size = (sizeof(struct flow_counter_stats) *
7431                         MLX5_COUNTERS_PER_POOL +
7432                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7433                         sizeof(struct mlx5_counter_stats_mem_mng);
7434         size_t pgsize = rte_mem_page_size();
7435         uint8_t *mem;
7436         int i;
7437
7438         if (pgsize == (size_t)-1) {
7439                 DRV_LOG(ERR, "Failed to get mem page size");
7440                 rte_errno = ENOMEM;
7441                 return -ENOMEM;
7442         }
7443         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7444         if (!mem) {
7445                 rte_errno = ENOMEM;
7446                 return -ENOMEM;
7447         }
7448         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7449         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7450         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7451                                                  IBV_ACCESS_LOCAL_WRITE);
7452         if (!mem_mng->umem) {
7453                 rte_errno = errno;
7454                 mlx5_free(mem);
7455                 return -rte_errno;
7456         }
7457         memset(&mkey_attr, 0, sizeof(mkey_attr));
7458         mkey_attr.addr = (uintptr_t)mem;
7459         mkey_attr.size = size;
7460         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7461         mkey_attr.pd = sh->pdn;
7462         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7463         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7464         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7465         if (!mem_mng->dm) {
7466                 mlx5_os_umem_dereg(mem_mng->umem);
7467                 rte_errno = errno;
7468                 mlx5_free(mem);
7469                 return -rte_errno;
7470         }
7471         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7472         raw_data = (volatile struct flow_counter_stats *)mem;
7473         for (i = 0; i < raws_n; ++i) {
7474                 mem_mng->raws[i].mem_mng = mem_mng;
7475                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7476         }
7477         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7478                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7479                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7480                                  next);
7481         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7482         sh->cmng.mem_mng = mem_mng;
7483         return 0;
7484 }
7485
7486 /**
7487  * Set the statistic memory to the new counter pool.
7488  *
7489  * @param[in] sh
7490  *   Pointer to mlx5_dev_ctx_shared object.
7491  * @param[in] pool
7492  *   Pointer to the pool to set the statistic memory.
7493  *
7494  * @return
7495  *   0 on success, a negative errno value otherwise.
7496  */
7497 static int
7498 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7499                                struct mlx5_flow_counter_pool *pool)
7500 {
7501         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7502         /* Resize statistic memory once used out. */
7503         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7504             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7505                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7506                 return -1;
7507         }
7508         rte_spinlock_lock(&pool->sl);
7509         pool->raw = cmng->mem_mng->raws + pool->index %
7510                     MLX5_CNT_CONTAINER_RESIZE;
7511         rte_spinlock_unlock(&pool->sl);
7512         pool->raw_hw = NULL;
7513         return 0;
7514 }
7515
7516 #define MLX5_POOL_QUERY_FREQ_US 1000000
7517
7518 /**
7519  * Set the periodic procedure for triggering asynchronous batch queries for all
7520  * the counter pools.
7521  *
7522  * @param[in] sh
7523  *   Pointer to mlx5_dev_ctx_shared object.
7524  */
7525 void
7526 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7527 {
7528         uint32_t pools_n, us;
7529
7530         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7531         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7532         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7533         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7534                 sh->cmng.query_thread_on = 0;
7535                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7536         } else {
7537                 sh->cmng.query_thread_on = 1;
7538         }
7539 }
7540
7541 /**
7542  * The periodic procedure for triggering asynchronous batch queries for all the
7543  * counter pools. This function is probably called by the host thread.
7544  *
7545  * @param[in] arg
7546  *   The parameter for the alarm process.
7547  */
7548 void
7549 mlx5_flow_query_alarm(void *arg)
7550 {
7551         struct mlx5_dev_ctx_shared *sh = arg;
7552         int ret;
7553         uint16_t pool_index = sh->cmng.pool_index;
7554         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7555         struct mlx5_flow_counter_pool *pool;
7556         uint16_t n_valid;
7557
7558         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7559                 goto set_alarm;
7560         rte_spinlock_lock(&cmng->pool_update_sl);
7561         pool = cmng->pools[pool_index];
7562         n_valid = cmng->n_valid;
7563         rte_spinlock_unlock(&cmng->pool_update_sl);
7564         /* Set the statistic memory to the new created pool. */
7565         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7566                 goto set_alarm;
7567         if (pool->raw_hw)
7568                 /* There is a pool query in progress. */
7569                 goto set_alarm;
7570         pool->raw_hw =
7571                 LIST_FIRST(&sh->cmng.free_stat_raws);
7572         if (!pool->raw_hw)
7573                 /* No free counter statistics raw memory. */
7574                 goto set_alarm;
7575         /*
7576          * Identify the counters released between query trigger and query
7577          * handle more efficiently. The counter released in this gap period
7578          * should wait for a new round of query as the new arrived packets
7579          * will not be taken into account.
7580          */
7581         pool->query_gen++;
7582         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7583                                                MLX5_COUNTERS_PER_POOL,
7584                                                NULL, NULL,
7585                                                pool->raw_hw->mem_mng->dm->id,
7586                                                (void *)(uintptr_t)
7587                                                pool->raw_hw->data,
7588                                                sh->devx_comp,
7589                                                (uint64_t)(uintptr_t)pool);
7590         if (ret) {
7591                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7592                         " %d", pool->min_dcs->id);
7593                 pool->raw_hw = NULL;
7594                 goto set_alarm;
7595         }
7596         LIST_REMOVE(pool->raw_hw, next);
7597         sh->cmng.pending_queries++;
7598         pool_index++;
7599         if (pool_index >= n_valid)
7600                 pool_index = 0;
7601 set_alarm:
7602         sh->cmng.pool_index = pool_index;
7603         mlx5_set_query_alarm(sh);
7604 }
7605
7606 /**
7607  * Check and callback event for new aged flow in the counter pool
7608  *
7609  * @param[in] sh
7610  *   Pointer to mlx5_dev_ctx_shared object.
7611  * @param[in] pool
7612  *   Pointer to Current counter pool.
7613  */
7614 static void
7615 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7616                    struct mlx5_flow_counter_pool *pool)
7617 {
7618         struct mlx5_priv *priv;
7619         struct mlx5_flow_counter *cnt;
7620         struct mlx5_age_info *age_info;
7621         struct mlx5_age_param *age_param;
7622         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7623         struct mlx5_counter_stats_raw *prev = pool->raw;
7624         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7625         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7626         uint16_t expected = AGE_CANDIDATE;
7627         uint32_t i;
7628
7629         pool->time_of_last_age_check = curr_time;
7630         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7631                 cnt = MLX5_POOL_GET_CNT(pool, i);
7632                 age_param = MLX5_CNT_TO_AGE(cnt);
7633                 if (__atomic_load_n(&age_param->state,
7634                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7635                         continue;
7636                 if (cur->data[i].hits != prev->data[i].hits) {
7637                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7638                                          __ATOMIC_RELAXED);
7639                         continue;
7640                 }
7641                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7642                                        time_delta,
7643                                        __ATOMIC_RELAXED) <= age_param->timeout)
7644                         continue;
7645                 /**
7646                  * Hold the lock first, or if between the
7647                  * state AGE_TMOUT and tailq operation the
7648                  * release happened, the release procedure
7649                  * may delete a non-existent tailq node.
7650                  */
7651                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7652                 age_info = GET_PORT_AGE_INFO(priv);
7653                 rte_spinlock_lock(&age_info->aged_sl);
7654                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7655                                                 AGE_TMOUT, false,
7656                                                 __ATOMIC_RELAXED,
7657                                                 __ATOMIC_RELAXED)) {
7658                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7659                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7660                 }
7661                 rte_spinlock_unlock(&age_info->aged_sl);
7662         }
7663         mlx5_age_event_prepare(sh);
7664 }
7665
7666 /**
7667  * Handler for the HW respond about ready values from an asynchronous batch
7668  * query. This function is probably called by the host thread.
7669  *
7670  * @param[in] sh
7671  *   The pointer to the shared device context.
7672  * @param[in] async_id
7673  *   The Devx async ID.
7674  * @param[in] status
7675  *   The status of the completion.
7676  */
7677 void
7678 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7679                                   uint64_t async_id, int status)
7680 {
7681         struct mlx5_flow_counter_pool *pool =
7682                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7683         struct mlx5_counter_stats_raw *raw_to_free;
7684         uint8_t query_gen = pool->query_gen ^ 1;
7685         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7686         enum mlx5_counter_type cnt_type =
7687                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7688                                 MLX5_COUNTER_TYPE_ORIGIN;
7689
7690         if (unlikely(status)) {
7691                 raw_to_free = pool->raw_hw;
7692         } else {
7693                 raw_to_free = pool->raw;
7694                 if (pool->is_aged)
7695                         mlx5_flow_aging_check(sh, pool);
7696                 rte_spinlock_lock(&pool->sl);
7697                 pool->raw = pool->raw_hw;
7698                 rte_spinlock_unlock(&pool->sl);
7699                 /* Be sure the new raw counters data is updated in memory. */
7700                 rte_io_wmb();
7701                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7702                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7703                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7704                                      &pool->counters[query_gen], next);
7705                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7706                 }
7707         }
7708         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7709         pool->raw_hw = NULL;
7710         sh->cmng.pending_queries--;
7711 }
7712
7713 static int
7714 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7715                     const struct flow_grp_info *grp_info,
7716                     struct rte_flow_error *error)
7717 {
7718         if (grp_info->transfer && grp_info->external &&
7719             grp_info->fdb_def_rule) {
7720                 if (group == UINT32_MAX)
7721                         return rte_flow_error_set
7722                                                 (error, EINVAL,
7723                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7724                                                  NULL,
7725                                                  "group index not supported");
7726                 *table = group + 1;
7727         } else {
7728                 *table = group;
7729         }
7730         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7731         return 0;
7732 }
7733
7734 /**
7735  * Translate the rte_flow group index to HW table value.
7736  *
7737  * If tunnel offload is disabled, all group ids converted to flow table
7738  * id using the standard method.
7739  * If tunnel offload is enabled, group id can be converted using the
7740  * standard or tunnel conversion method. Group conversion method
7741  * selection depends on flags in `grp_info` parameter:
7742  * - Internal (grp_info.external == 0) groups conversion uses the
7743  *   standard method.
7744  * - Group ids in JUMP action converted with the tunnel conversion.
7745  * - Group id in rule attribute conversion depends on a rule type and
7746  *   group id value:
7747  *   ** non zero group attributes converted with the tunnel method
7748  *   ** zero group attribute in non-tunnel rule is converted using the
7749  *      standard method - there's only one root table
7750  *   ** zero group attribute in steer tunnel rule is converted with the
7751  *      standard method - single root table
7752  *   ** zero group attribute in match tunnel rule is a special OvS
7753  *      case: that value is used for portability reasons. That group
7754  *      id is converted with the tunnel conversion method.
7755  *
7756  * @param[in] dev
7757  *   Port device
7758  * @param[in] tunnel
7759  *   PMD tunnel offload object
7760  * @param[in] group
7761  *   rte_flow group index value.
7762  * @param[out] table
7763  *   HW table value.
7764  * @param[in] grp_info
7765  *   flags used for conversion
7766  * @param[out] error
7767  *   Pointer to error structure.
7768  *
7769  * @return
7770  *   0 on success, a negative errno value otherwise and rte_errno is set.
7771  */
7772 int
7773 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7774                          const struct mlx5_flow_tunnel *tunnel,
7775                          uint32_t group, uint32_t *table,
7776                          const struct flow_grp_info *grp_info,
7777                          struct rte_flow_error *error)
7778 {
7779         int ret;
7780         bool standard_translation;
7781
7782         if (!grp_info->skip_scale && grp_info->external &&
7783             group < MLX5_MAX_TABLES_EXTERNAL)
7784                 group *= MLX5_FLOW_TABLE_FACTOR;
7785         if (is_tunnel_offload_active(dev)) {
7786                 standard_translation = !grp_info->external ||
7787                                         grp_info->std_tbl_fix;
7788         } else {
7789                 standard_translation = true;
7790         }
7791         DRV_LOG(DEBUG,
7792                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7793                 dev->data->port_id, group, grp_info->transfer,
7794                 grp_info->external, grp_info->fdb_def_rule,
7795                 standard_translation ? "STANDARD" : "TUNNEL");
7796         if (standard_translation)
7797                 ret = flow_group_to_table(dev->data->port_id, group, table,
7798                                           grp_info, error);
7799         else
7800                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7801                                                       table, error);
7802
7803         return ret;
7804 }
7805
7806 /**
7807  * Discover availability of metadata reg_c's.
7808  *
7809  * Iteratively use test flows to check availability.
7810  *
7811  * @param[in] dev
7812  *   Pointer to the Ethernet device structure.
7813  *
7814  * @return
7815  *   0 on success, a negative errno value otherwise and rte_errno is set.
7816  */
7817 int
7818 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7819 {
7820         struct mlx5_priv *priv = dev->data->dev_private;
7821         struct mlx5_dev_config *config = &priv->config;
7822         enum modify_reg idx;
7823         int n = 0;
7824
7825         /* reg_c[0] and reg_c[1] are reserved. */
7826         config->flow_mreg_c[n++] = REG_C_0;
7827         config->flow_mreg_c[n++] = REG_C_1;
7828         /* Discover availability of other reg_c's. */
7829         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7830                 struct rte_flow_attr attr = {
7831                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7832                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7833                         .ingress = 1,
7834                 };
7835                 struct rte_flow_item items[] = {
7836                         [0] = {
7837                                 .type = RTE_FLOW_ITEM_TYPE_END,
7838                         },
7839                 };
7840                 struct rte_flow_action actions[] = {
7841                         [0] = {
7842                                 .type = (enum rte_flow_action_type)
7843                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
7844                                 .conf = &(struct mlx5_flow_action_copy_mreg){
7845                                         .src = REG_C_1,
7846                                         .dst = idx,
7847                                 },
7848                         },
7849                         [1] = {
7850                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7851                                 .conf = &(struct rte_flow_action_jump){
7852                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
7853                                 },
7854                         },
7855                         [2] = {
7856                                 .type = RTE_FLOW_ACTION_TYPE_END,
7857                         },
7858                 };
7859                 uint32_t flow_idx;
7860                 struct rte_flow *flow;
7861                 struct rte_flow_error error;
7862
7863                 if (!config->dv_flow_en)
7864                         break;
7865                 /* Create internal flow, validation skips copy action. */
7866                 flow_idx = flow_list_create(dev, NULL, &attr, items,
7867                                             actions, false, &error);
7868                 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
7869                                       flow_idx);
7870                 if (!flow)
7871                         continue;
7872                 config->flow_mreg_c[n++] = idx;
7873                 flow_list_destroy(dev, NULL, flow_idx);
7874         }
7875         for (; n < MLX5_MREG_C_NUM; ++n)
7876                 config->flow_mreg_c[n] = REG_NON;
7877         return 0;
7878 }
7879
7880 /**
7881  * Dump flow raw hw data to file
7882  *
7883  * @param[in] dev
7884  *    The pointer to Ethernet device.
7885  * @param[in] file
7886  *   A pointer to a file for output.
7887  * @param[out] error
7888  *   Perform verbose error reporting if not NULL. PMDs initialize this
7889  *   structure in case of error only.
7890  * @return
7891  *   0 on success, a nagative value otherwise.
7892  */
7893 int
7894 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
7895                    FILE *file,
7896                    struct rte_flow_error *error __rte_unused)
7897 {
7898         struct mlx5_priv *priv = dev->data->dev_private;
7899         struct mlx5_dev_ctx_shared *sh = priv->sh;
7900         uint32_t handle_idx;
7901         int ret;
7902         struct mlx5_flow_handle *dh;
7903         struct rte_flow *flow;
7904
7905         if (!priv->config.dv_flow_en) {
7906                 if (fputs("device dv flow disabled\n", file) <= 0)
7907                         return -errno;
7908                 return -ENOTSUP;
7909         }
7910
7911         /* dump all */
7912         if (!flow_idx)
7913                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
7914                                         sh->rx_domain,
7915                                         sh->tx_domain, file);
7916         /* dump one */
7917         flow = mlx5_ipool_get(priv->sh->ipool
7918                         [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
7919         if (!flow)
7920                 return -ENOENT;
7921
7922         handle_idx = flow->dev_handles;
7923         while (handle_idx) {
7924                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7925                                 handle_idx);
7926                 if (!dh)
7927                         return -ENOENT;
7928                 if (dh->drv_flow) {
7929                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
7930                                         file);
7931                         if (ret)
7932                                 return -ENOENT;
7933                 }
7934                 handle_idx = dh->next.next;
7935         }
7936         return 0;
7937 }
7938
7939 /**
7940  * Get aged-out flows.
7941  *
7942  * @param[in] dev
7943  *   Pointer to the Ethernet device structure.
7944  * @param[in] context
7945  *   The address of an array of pointers to the aged-out flows contexts.
7946  * @param[in] nb_countexts
7947  *   The length of context array pointers.
7948  * @param[out] error
7949  *   Perform verbose error reporting if not NULL. Initialized in case of
7950  *   error only.
7951  *
7952  * @return
7953  *   how many contexts get in success, otherwise negative errno value.
7954  *   if nb_contexts is 0, return the amount of all aged contexts.
7955  *   if nb_contexts is not 0 , return the amount of aged flows reported
7956  *   in the context array.
7957  */
7958 int
7959 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
7960                         uint32_t nb_contexts, struct rte_flow_error *error)
7961 {
7962         const struct mlx5_flow_driver_ops *fops;
7963         struct rte_flow_attr attr = { .transfer = 0 };
7964
7965         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7966                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7967                 return fops->get_aged_flows(dev, contexts, nb_contexts,
7968                                                     error);
7969         }
7970         DRV_LOG(ERR,
7971                 "port %u get aged flows is not supported.",
7972                  dev->data->port_id);
7973         return -ENOTSUP;
7974 }
7975
7976 /* Wrapper for driver action_validate op callback */
7977 static int
7978 flow_drv_action_validate(struct rte_eth_dev *dev,
7979                          const struct rte_flow_indir_action_conf *conf,
7980                          const struct rte_flow_action *action,
7981                          const struct mlx5_flow_driver_ops *fops,
7982                          struct rte_flow_error *error)
7983 {
7984         static const char err_msg[] = "indirect action validation unsupported";
7985
7986         if (!fops->action_validate) {
7987                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
7988                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
7989                                    NULL, err_msg);
7990                 return -rte_errno;
7991         }
7992         return fops->action_validate(dev, conf, action, error);
7993 }
7994
7995 /**
7996  * Destroys the shared action by handle.
7997  *
7998  * @param dev
7999  *   Pointer to Ethernet device structure.
8000  * @param[in] handle
8001  *   Handle for the indirect action object to be destroyed.
8002  * @param[out] error
8003  *   Perform verbose error reporting if not NULL. PMDs initialize this
8004  *   structure in case of error only.
8005  *
8006  * @return
8007  *   0 on success, a negative errno value otherwise and rte_errno is set.
8008  *
8009  * @note: wrapper for driver action_create op callback.
8010  */
8011 static int
8012 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
8013                            struct rte_flow_action_handle *handle,
8014                            struct rte_flow_error *error)
8015 {
8016         static const char err_msg[] = "indirect action destruction unsupported";
8017         struct rte_flow_attr attr = { .transfer = 0 };
8018         const struct mlx5_flow_driver_ops *fops =
8019                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8020
8021         if (!fops->action_destroy) {
8022                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8023                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8024                                    NULL, err_msg);
8025                 return -rte_errno;
8026         }
8027         return fops->action_destroy(dev, handle, error);
8028 }
8029
8030 /* Wrapper for driver action_destroy op callback */
8031 static int
8032 flow_drv_action_update(struct rte_eth_dev *dev,
8033                        struct rte_flow_action_handle *handle,
8034                        const void *update,
8035                        const struct mlx5_flow_driver_ops *fops,
8036                        struct rte_flow_error *error)
8037 {
8038         static const char err_msg[] = "indirect action update unsupported";
8039
8040         if (!fops->action_update) {
8041                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8042                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8043                                    NULL, err_msg);
8044                 return -rte_errno;
8045         }
8046         return fops->action_update(dev, handle, update, error);
8047 }
8048
8049 /* Wrapper for driver action_destroy op callback */
8050 static int
8051 flow_drv_action_query(struct rte_eth_dev *dev,
8052                       const struct rte_flow_action_handle *handle,
8053                       void *data,
8054                       const struct mlx5_flow_driver_ops *fops,
8055                       struct rte_flow_error *error)
8056 {
8057         static const char err_msg[] = "indirect action query unsupported";
8058
8059         if (!fops->action_query) {
8060                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8061                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8062                                    NULL, err_msg);
8063                 return -rte_errno;
8064         }
8065         return fops->action_query(dev, handle, data, error);
8066 }
8067
8068 /**
8069  * Create indirect action for reuse in multiple flow rules.
8070  *
8071  * @param dev
8072  *   Pointer to Ethernet device structure.
8073  * @param conf
8074  *   Pointer to indirect action object configuration.
8075  * @param[in] action
8076  *   Action configuration for indirect action object creation.
8077  * @param[out] error
8078  *   Perform verbose error reporting if not NULL. PMDs initialize this
8079  *   structure in case of error only.
8080  * @return
8081  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8082  */
8083 static struct rte_flow_action_handle *
8084 mlx5_action_handle_create(struct rte_eth_dev *dev,
8085                           const struct rte_flow_indir_action_conf *conf,
8086                           const struct rte_flow_action *action,
8087                           struct rte_flow_error *error)
8088 {
8089         static const char err_msg[] = "indirect action creation unsupported";
8090         struct rte_flow_attr attr = { .transfer = 0 };
8091         const struct mlx5_flow_driver_ops *fops =
8092                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8093
8094         if (flow_drv_action_validate(dev, conf, action, fops, error))
8095                 return NULL;
8096         if (!fops->action_create) {
8097                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8098                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8099                                    NULL, err_msg);
8100                 return NULL;
8101         }
8102         return fops->action_create(dev, conf, action, error);
8103 }
8104
8105 /**
8106  * Updates inplace the indirect action configuration pointed by *handle*
8107  * with the configuration provided as *update* argument.
8108  * The update of the indirect action configuration effects all flow rules
8109  * reusing the action via handle.
8110  *
8111  * @param dev
8112  *   Pointer to Ethernet device structure.
8113  * @param[in] handle
8114  *   Handle for the indirect action to be updated.
8115  * @param[in] update
8116  *   Action specification used to modify the action pointed by handle.
8117  *   *update* could be of same type with the action pointed by the *handle*
8118  *   handle argument, or some other structures like a wrapper, depending on
8119  *   the indirect action type.
8120  * @param[out] error
8121  *   Perform verbose error reporting if not NULL. PMDs initialize this
8122  *   structure in case of error only.
8123  *
8124  * @return
8125  *   0 on success, a negative errno value otherwise and rte_errno is set.
8126  */
8127 static int
8128 mlx5_action_handle_update(struct rte_eth_dev *dev,
8129                 struct rte_flow_action_handle *handle,
8130                 const void *update,
8131                 struct rte_flow_error *error)
8132 {
8133         struct rte_flow_attr attr = { .transfer = 0 };
8134         const struct mlx5_flow_driver_ops *fops =
8135                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8136         int ret;
8137
8138         ret = flow_drv_action_validate(dev, NULL,
8139                         (const struct rte_flow_action *)update, fops, error);
8140         if (ret)
8141                 return ret;
8142         return flow_drv_action_update(dev, handle, update, fops,
8143                                       error);
8144 }
8145
8146 /**
8147  * Query the indirect action by handle.
8148  *
8149  * This function allows retrieving action-specific data such as counters.
8150  * Data is gathered by special action which may be present/referenced in
8151  * more than one flow rule definition.
8152  *
8153  * see @RTE_FLOW_ACTION_TYPE_COUNT
8154  *
8155  * @param dev
8156  *   Pointer to Ethernet device structure.
8157  * @param[in] handle
8158  *   Handle for the indirect action to query.
8159  * @param[in, out] data
8160  *   Pointer to storage for the associated query data type.
8161  * @param[out] error
8162  *   Perform verbose error reporting if not NULL. PMDs initialize this
8163  *   structure in case of error only.
8164  *
8165  * @return
8166  *   0 on success, a negative errno value otherwise and rte_errno is set.
8167  */
8168 static int
8169 mlx5_action_handle_query(struct rte_eth_dev *dev,
8170                          const struct rte_flow_action_handle *handle,
8171                          void *data,
8172                          struct rte_flow_error *error)
8173 {
8174         struct rte_flow_attr attr = { .transfer = 0 };
8175         const struct mlx5_flow_driver_ops *fops =
8176                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8177
8178         return flow_drv_action_query(dev, handle, data, fops, error);
8179 }
8180
8181 /**
8182  * Destroy all indirect actions (shared RSS).
8183  *
8184  * @param dev
8185  *   Pointer to Ethernet device.
8186  *
8187  * @return
8188  *   0 on success, a negative errno value otherwise and rte_errno is set.
8189  */
8190 int
8191 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8192 {
8193         struct rte_flow_error error;
8194         struct mlx5_priv *priv = dev->data->dev_private;
8195         struct mlx5_shared_action_rss *shared_rss;
8196         int ret = 0;
8197         uint32_t idx;
8198
8199         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8200                       priv->rss_shared_actions, idx, shared_rss, next) {
8201                 ret |= mlx5_action_handle_destroy(dev,
8202                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8203         }
8204         return ret;
8205 }
8206
8207 #ifndef HAVE_MLX5DV_DR
8208 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8209 #else
8210 #define MLX5_DOMAIN_SYNC_FLOW \
8211         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8212 #endif
8213
8214 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8215 {
8216         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8217         const struct mlx5_flow_driver_ops *fops;
8218         int ret;
8219         struct rte_flow_attr attr = { .transfer = 0 };
8220
8221         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8222         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8223         if (ret > 0)
8224                 ret = -ret;
8225         return ret;
8226 }
8227
8228 const struct mlx5_flow_tunnel *
8229 mlx5_get_tof(const struct rte_flow_item *item,
8230              const struct rte_flow_action *action,
8231              enum mlx5_tof_rule_type *rule_type)
8232 {
8233         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8234                 if (item->type == (typeof(item->type))
8235                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8236                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8237                         return flow_items_to_tunnel(item);
8238                 }
8239         }
8240         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8241                 if (action->type == (typeof(action->type))
8242                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8243                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8244                         return flow_actions_to_tunnel(action);
8245                 }
8246         }
8247         return NULL;
8248 }
8249
8250 /**
8251  * tunnel offload functionalilty is defined for DV environment only
8252  */
8253 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8254 __extension__
8255 union tunnel_offload_mark {
8256         uint32_t val;
8257         struct {
8258                 uint32_t app_reserve:8;
8259                 uint32_t table_id:15;
8260                 uint32_t transfer:1;
8261                 uint32_t _unused_:8;
8262         };
8263 };
8264
8265 static bool
8266 mlx5_access_tunnel_offload_db
8267         (struct rte_eth_dev *dev,
8268          bool (*match)(struct rte_eth_dev *,
8269                        struct mlx5_flow_tunnel *, const void *),
8270          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8271          void (*miss)(struct rte_eth_dev *, void *),
8272          void *ctx, bool lock_op);
8273
8274 static int
8275 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8276                              struct rte_flow *flow,
8277                              const struct rte_flow_attr *attr,
8278                              const struct rte_flow_action *app_actions,
8279                              uint32_t flow_idx,
8280                              const struct mlx5_flow_tunnel *tunnel,
8281                              struct tunnel_default_miss_ctx *ctx,
8282                              struct rte_flow_error *error)
8283 {
8284         struct mlx5_priv *priv = dev->data->dev_private;
8285         struct mlx5_flow *dev_flow;
8286         struct rte_flow_attr miss_attr = *attr;
8287         const struct rte_flow_item miss_items[2] = {
8288                 {
8289                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8290                         .spec = NULL,
8291                         .last = NULL,
8292                         .mask = NULL
8293                 },
8294                 {
8295                         .type = RTE_FLOW_ITEM_TYPE_END,
8296                         .spec = NULL,
8297                         .last = NULL,
8298                         .mask = NULL
8299                 }
8300         };
8301         union tunnel_offload_mark mark_id;
8302         struct rte_flow_action_mark miss_mark;
8303         struct rte_flow_action miss_actions[3] = {
8304                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8305                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8306         };
8307         const struct rte_flow_action_jump *jump_data;
8308         uint32_t i, flow_table = 0; /* prevent compilation warning */
8309         struct flow_grp_info grp_info = {
8310                 .external = 1,
8311                 .transfer = attr->transfer,
8312                 .fdb_def_rule = !!priv->fdb_def_rule,
8313                 .std_tbl_fix = 0,
8314         };
8315         int ret;
8316
8317         if (!attr->transfer) {
8318                 uint32_t q_size;
8319
8320                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8321                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8322                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8323                                          0, SOCKET_ID_ANY);
8324                 if (!ctx->queue)
8325                         return rte_flow_error_set
8326                                 (error, ENOMEM,
8327                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8328                                 NULL, "invalid default miss RSS");
8329                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8330                 ctx->action_rss.level = 0,
8331                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8332                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8333                 ctx->action_rss.queue_num = priv->reta_idx_n,
8334                 ctx->action_rss.key = priv->rss_conf.rss_key,
8335                 ctx->action_rss.queue = ctx->queue;
8336                 if (!priv->reta_idx_n || !priv->rxqs_n)
8337                         return rte_flow_error_set
8338                                 (error, EINVAL,
8339                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8340                                 NULL, "invalid port configuration");
8341                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8342                         ctx->action_rss.types = 0;
8343                 for (i = 0; i != priv->reta_idx_n; ++i)
8344                         ctx->queue[i] = (*priv->reta_idx)[i];
8345         } else {
8346                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8347                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8348         }
8349         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8350         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8351         jump_data = app_actions->conf;
8352         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8353         miss_attr.group = jump_data->group;
8354         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8355                                        &flow_table, &grp_info, error);
8356         if (ret)
8357                 return rte_flow_error_set(error, EINVAL,
8358                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8359                                           NULL, "invalid tunnel id");
8360         mark_id.app_reserve = 0;
8361         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8362         mark_id.transfer = !!attr->transfer;
8363         mark_id._unused_ = 0;
8364         miss_mark.id = mark_id.val;
8365         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8366                                     miss_items, miss_actions, flow_idx, error);
8367         if (!dev_flow)
8368                 return -rte_errno;
8369         dev_flow->flow = flow;
8370         dev_flow->external = true;
8371         dev_flow->tunnel = tunnel;
8372         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8373         /* Subflow object was created, we must include one in the list. */
8374         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8375                       dev_flow->handle, next);
8376         DRV_LOG(DEBUG,
8377                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8378                 dev->data->port_id, tunnel->app_tunnel.type,
8379                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8380         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8381                                   miss_actions, error);
8382         if (!ret)
8383                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8384                                                   error);
8385
8386         return ret;
8387 }
8388
8389 static const struct mlx5_flow_tbl_data_entry  *
8390 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8391 {
8392         struct mlx5_priv *priv = dev->data->dev_private;
8393         struct mlx5_dev_ctx_shared *sh = priv->sh;
8394         struct mlx5_hlist_entry *he;
8395         union tunnel_offload_mark mbits = { .val = mark };
8396         union mlx5_flow_tbl_key table_key = {
8397                 {
8398                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8399                         .id = 0,
8400                         .reserved = 0,
8401                         .dummy = 0,
8402                         .is_fdb = !!mbits.transfer,
8403                         .is_egress = 0,
8404                 }
8405         };
8406         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
8407         return he ?
8408                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8409 }
8410
8411 static void
8412 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
8413                                    struct mlx5_hlist_entry *entry)
8414 {
8415         struct mlx5_dev_ctx_shared *sh = list->ctx;
8416         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8417
8418         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8419                         tunnel_flow_tbl_to_id(tte->flow_table));
8420         mlx5_free(tte);
8421 }
8422
8423 static int
8424 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
8425                                   struct mlx5_hlist_entry *entry,
8426                                   uint64_t key, void *cb_ctx __rte_unused)
8427 {
8428         union tunnel_tbl_key tbl = {
8429                 .val = key,
8430         };
8431         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8432
8433         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8434 }
8435
8436 static struct mlx5_hlist_entry *
8437 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
8438                                    void *ctx __rte_unused)
8439 {
8440         struct mlx5_dev_ctx_shared *sh = list->ctx;
8441         struct tunnel_tbl_entry *tte;
8442         union tunnel_tbl_key tbl = {
8443                 .val = key,
8444         };
8445
8446         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8447                           sizeof(*tte), 0,
8448                           SOCKET_ID_ANY);
8449         if (!tte)
8450                 goto err;
8451         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8452                           &tte->flow_table);
8453         if (tte->flow_table >= MLX5_MAX_TABLES) {
8454                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8455                         tte->flow_table);
8456                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8457                                 tte->flow_table);
8458                 goto err;
8459         } else if (!tte->flow_table) {
8460                 goto err;
8461         }
8462         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8463         tte->tunnel_id = tbl.tunnel_id;
8464         tte->group = tbl.group;
8465         return &tte->hash;
8466 err:
8467         if (tte)
8468                 mlx5_free(tte);
8469         return NULL;
8470 }
8471
8472 static uint32_t
8473 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8474                                 const struct mlx5_flow_tunnel *tunnel,
8475                                 uint32_t group, uint32_t *table,
8476                                 struct rte_flow_error *error)
8477 {
8478         struct mlx5_hlist_entry *he;
8479         struct tunnel_tbl_entry *tte;
8480         union tunnel_tbl_key key = {
8481                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8482                 .group = group
8483         };
8484         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8485         struct mlx5_hlist *group_hash;
8486
8487         group_hash = tunnel ? tunnel->groups : thub->groups;
8488         he = mlx5_hlist_register(group_hash, key.val, NULL);
8489         if (!he)
8490                 return rte_flow_error_set(error, EINVAL,
8491                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8492                                           NULL,
8493                                           "tunnel group index not supported");
8494         tte = container_of(he, typeof(*tte), hash);
8495         *table = tte->flow_table;
8496         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8497                 dev->data->port_id, key.tunnel_id, group, *table);
8498         return 0;
8499 }
8500
8501 static void
8502 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8503                       struct mlx5_flow_tunnel *tunnel)
8504 {
8505         struct mlx5_priv *priv = dev->data->dev_private;
8506         struct mlx5_indexed_pool *ipool;
8507
8508         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8509                 dev->data->port_id, tunnel->tunnel_id);
8510         LIST_REMOVE(tunnel, chain);
8511         mlx5_hlist_destroy(tunnel->groups);
8512         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8513         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8514 }
8515
8516 static bool
8517 mlx5_access_tunnel_offload_db
8518         (struct rte_eth_dev *dev,
8519          bool (*match)(struct rte_eth_dev *,
8520                        struct mlx5_flow_tunnel *, const void *),
8521          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8522          void (*miss)(struct rte_eth_dev *, void *),
8523          void *ctx, bool lock_op)
8524 {
8525         bool verdict = false;
8526         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8527         struct mlx5_flow_tunnel *tunnel;
8528
8529         rte_spinlock_lock(&thub->sl);
8530         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8531                 verdict = match(dev, tunnel, (const void *)ctx);
8532                 if (verdict)
8533                         break;
8534         }
8535         if (!lock_op)
8536                 rte_spinlock_unlock(&thub->sl);
8537         if (verdict && hit)
8538                 hit(dev, tunnel, ctx);
8539         if (!verdict && miss)
8540                 miss(dev, ctx);
8541         if (lock_op)
8542                 rte_spinlock_unlock(&thub->sl);
8543
8544         return verdict;
8545 }
8546
8547 struct tunnel_db_find_tunnel_id_ctx {
8548         uint32_t tunnel_id;
8549         struct mlx5_flow_tunnel *tunnel;
8550 };
8551
8552 static bool
8553 find_tunnel_id_match(struct rte_eth_dev *dev,
8554                      struct mlx5_flow_tunnel *tunnel, const void *x)
8555 {
8556         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8557
8558         RTE_SET_USED(dev);
8559         return tunnel->tunnel_id == ctx->tunnel_id;
8560 }
8561
8562 static void
8563 find_tunnel_id_hit(struct rte_eth_dev *dev,
8564                    struct mlx5_flow_tunnel *tunnel, void *x)
8565 {
8566         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8567         RTE_SET_USED(dev);
8568         ctx->tunnel = tunnel;
8569 }
8570
8571 static struct mlx5_flow_tunnel *
8572 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8573 {
8574         struct tunnel_db_find_tunnel_id_ctx ctx = {
8575                 .tunnel_id = id,
8576         };
8577
8578         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8579                                       find_tunnel_id_hit, NULL, &ctx, true);
8580
8581         return ctx.tunnel;
8582 }
8583
8584 static struct mlx5_flow_tunnel *
8585 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8586                           const struct rte_flow_tunnel *app_tunnel)
8587 {
8588         struct mlx5_priv *priv = dev->data->dev_private;
8589         struct mlx5_indexed_pool *ipool;
8590         struct mlx5_flow_tunnel *tunnel;
8591         uint32_t id;
8592
8593         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8594         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8595         if (!tunnel)
8596                 return NULL;
8597         if (id >= MLX5_MAX_TUNNELS) {
8598                 mlx5_ipool_free(ipool, id);
8599                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8600                 return NULL;
8601         }
8602         tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
8603                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8604                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8605                                            mlx5_flow_tunnel_grp2tbl_remove_cb);
8606         if (!tunnel->groups) {
8607                 mlx5_ipool_free(ipool, id);
8608                 return NULL;
8609         }
8610         tunnel->groups->ctx = priv->sh;
8611         /* initiate new PMD tunnel */
8612         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8613         tunnel->tunnel_id = id;
8614         tunnel->action.type = (typeof(tunnel->action.type))
8615                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8616         tunnel->action.conf = tunnel;
8617         tunnel->item.type = (typeof(tunnel->item.type))
8618                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8619         tunnel->item.spec = tunnel;
8620         tunnel->item.last = NULL;
8621         tunnel->item.mask = NULL;
8622
8623         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8624                 dev->data->port_id, tunnel->tunnel_id);
8625
8626         return tunnel;
8627 }
8628
8629 struct tunnel_db_get_tunnel_ctx {
8630         const struct rte_flow_tunnel *app_tunnel;
8631         struct mlx5_flow_tunnel *tunnel;
8632 };
8633
8634 static bool get_tunnel_match(struct rte_eth_dev *dev,
8635                              struct mlx5_flow_tunnel *tunnel, const void *x)
8636 {
8637         const struct tunnel_db_get_tunnel_ctx *ctx = x;
8638
8639         RTE_SET_USED(dev);
8640         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
8641                        sizeof(*ctx->app_tunnel));
8642 }
8643
8644 static void get_tunnel_hit(struct rte_eth_dev *dev,
8645                            struct mlx5_flow_tunnel *tunnel, void *x)
8646 {
8647         /* called under tunnel spinlock protection */
8648         struct tunnel_db_get_tunnel_ctx *ctx = x;
8649
8650         RTE_SET_USED(dev);
8651         tunnel->refctn++;
8652         ctx->tunnel = tunnel;
8653 }
8654
8655 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
8656 {
8657         /* called under tunnel spinlock protection */
8658         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8659         struct tunnel_db_get_tunnel_ctx *ctx = x;
8660
8661         rte_spinlock_unlock(&thub->sl);
8662         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
8663         rte_spinlock_lock(&thub->sl);
8664         if (ctx->tunnel) {
8665                 ctx->tunnel->refctn = 1;
8666                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
8667         }
8668 }
8669
8670
8671 static int
8672 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
8673                      const struct rte_flow_tunnel *app_tunnel,
8674                      struct mlx5_flow_tunnel **tunnel)
8675 {
8676         struct tunnel_db_get_tunnel_ctx ctx = {
8677                 .app_tunnel = app_tunnel,
8678         };
8679
8680         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
8681                                       get_tunnel_miss, &ctx, true);
8682         *tunnel = ctx.tunnel;
8683         return ctx.tunnel ? 0 : -ENOMEM;
8684 }
8685
8686 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
8687 {
8688         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
8689
8690         if (!thub)
8691                 return;
8692         if (!LIST_EMPTY(&thub->tunnels))
8693                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
8694         mlx5_hlist_destroy(thub->groups);
8695         mlx5_free(thub);
8696 }
8697
8698 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
8699 {
8700         int err;
8701         struct mlx5_flow_tunnel_hub *thub;
8702
8703         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
8704                            0, SOCKET_ID_ANY);
8705         if (!thub)
8706                 return -ENOMEM;
8707         LIST_INIT(&thub->tunnels);
8708         rte_spinlock_init(&thub->sl);
8709         thub->groups = mlx5_hlist_create("flow groups",
8710                                          rte_align32pow2(MLX5_MAX_TABLES), 0,
8711                                          0, mlx5_flow_tunnel_grp2tbl_create_cb,
8712                                          mlx5_flow_tunnel_grp2tbl_match_cb,
8713                                          mlx5_flow_tunnel_grp2tbl_remove_cb);
8714         if (!thub->groups) {
8715                 err = -rte_errno;
8716                 goto err;
8717         }
8718         thub->groups->ctx = sh;
8719         sh->tunnel_hub = thub;
8720
8721         return 0;
8722
8723 err:
8724         if (thub->groups)
8725                 mlx5_hlist_destroy(thub->groups);
8726         if (thub)
8727                 mlx5_free(thub);
8728         return err;
8729 }
8730
8731 static inline bool
8732 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
8733                           struct rte_flow_tunnel *tunnel,
8734                           const char *err_msg)
8735 {
8736         err_msg = NULL;
8737         if (!is_tunnel_offload_active(dev)) {
8738                 err_msg = "tunnel offload was not activated";
8739                 goto out;
8740         } else if (!tunnel) {
8741                 err_msg = "no application tunnel";
8742                 goto out;
8743         }
8744
8745         switch (tunnel->type) {
8746         default:
8747                 err_msg = "unsupported tunnel type";
8748                 goto out;
8749         case RTE_FLOW_ITEM_TYPE_VXLAN:
8750                 break;
8751         }
8752
8753 out:
8754         return !err_msg;
8755 }
8756
8757 static int
8758 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
8759                     struct rte_flow_tunnel *app_tunnel,
8760                     struct rte_flow_action **actions,
8761                     uint32_t *num_of_actions,
8762                     struct rte_flow_error *error)
8763 {
8764         int ret;
8765         struct mlx5_flow_tunnel *tunnel;
8766         const char *err_msg = NULL;
8767         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8768
8769         if (!verdict)
8770                 return rte_flow_error_set(error, EINVAL,
8771                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8772                                           err_msg);
8773         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8774         if (ret < 0) {
8775                 return rte_flow_error_set(error, ret,
8776                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
8777                                           "failed to initialize pmd tunnel");
8778         }
8779         *actions = &tunnel->action;
8780         *num_of_actions = 1;
8781         return 0;
8782 }
8783
8784 static int
8785 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
8786                        struct rte_flow_tunnel *app_tunnel,
8787                        struct rte_flow_item **items,
8788                        uint32_t *num_of_items,
8789                        struct rte_flow_error *error)
8790 {
8791         int ret;
8792         struct mlx5_flow_tunnel *tunnel;
8793         const char *err_msg = NULL;
8794         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
8795
8796         if (!verdict)
8797                 return rte_flow_error_set(error, EINVAL,
8798                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8799                                           err_msg);
8800         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
8801         if (ret < 0) {
8802                 return rte_flow_error_set(error, ret,
8803                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8804                                           "failed to initialize pmd tunnel");
8805         }
8806         *items = &tunnel->item;
8807         *num_of_items = 1;
8808         return 0;
8809 }
8810
8811 struct tunnel_db_element_release_ctx {
8812         struct rte_flow_item *items;
8813         struct rte_flow_action *actions;
8814         uint32_t num_elements;
8815         struct rte_flow_error *error;
8816         int ret;
8817 };
8818
8819 static bool
8820 tunnel_element_release_match(struct rte_eth_dev *dev,
8821                              struct mlx5_flow_tunnel *tunnel, const void *x)
8822 {
8823         const struct tunnel_db_element_release_ctx *ctx = x;
8824
8825         RTE_SET_USED(dev);
8826         if (ctx->num_elements != 1)
8827                 return false;
8828         else if (ctx->items)
8829                 return ctx->items == &tunnel->item;
8830         else if (ctx->actions)
8831                 return ctx->actions == &tunnel->action;
8832
8833         return false;
8834 }
8835
8836 static void
8837 tunnel_element_release_hit(struct rte_eth_dev *dev,
8838                            struct mlx5_flow_tunnel *tunnel, void *x)
8839 {
8840         struct tunnel_db_element_release_ctx *ctx = x;
8841         ctx->ret = 0;
8842         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
8843                 mlx5_flow_tunnel_free(dev, tunnel);
8844 }
8845
8846 static void
8847 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
8848 {
8849         struct tunnel_db_element_release_ctx *ctx = x;
8850         RTE_SET_USED(dev);
8851         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
8852                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
8853                                       "invalid argument");
8854 }
8855
8856 static int
8857 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
8858                        struct rte_flow_item *pmd_items,
8859                        uint32_t num_items, struct rte_flow_error *err)
8860 {
8861         struct tunnel_db_element_release_ctx ctx = {
8862                 .items = pmd_items,
8863                 .actions = NULL,
8864                 .num_elements = num_items,
8865                 .error = err,
8866         };
8867
8868         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8869                                       tunnel_element_release_hit,
8870                                       tunnel_element_release_miss, &ctx, false);
8871
8872         return ctx.ret;
8873 }
8874
8875 static int
8876 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
8877                          struct rte_flow_action *pmd_actions,
8878                          uint32_t num_actions, struct rte_flow_error *err)
8879 {
8880         struct tunnel_db_element_release_ctx ctx = {
8881                 .items = NULL,
8882                 .actions = pmd_actions,
8883                 .num_elements = num_actions,
8884                 .error = err,
8885         };
8886
8887         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
8888                                       tunnel_element_release_hit,
8889                                       tunnel_element_release_miss, &ctx, false);
8890
8891         return ctx.ret;
8892 }
8893
8894 static int
8895 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
8896                                   struct rte_mbuf *m,
8897                                   struct rte_flow_restore_info *info,
8898                                   struct rte_flow_error *err)
8899 {
8900         uint64_t ol_flags = m->ol_flags;
8901         const struct mlx5_flow_tbl_data_entry *tble;
8902         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
8903
8904         if (!is_tunnel_offload_active(dev)) {
8905                 info->flags = 0;
8906                 return 0;
8907         }
8908
8909         if ((ol_flags & mask) != mask)
8910                 goto err;
8911         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
8912         if (!tble) {
8913                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
8914                         dev->data->port_id, m->hash.fdir.hi);
8915                 goto err;
8916         }
8917         MLX5_ASSERT(tble->tunnel);
8918         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
8919         info->group_id = tble->group_id;
8920         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
8921                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
8922                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
8923
8924         return 0;
8925
8926 err:
8927         return rte_flow_error_set(err, EINVAL,
8928                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8929                                   "failed to get restore info");
8930 }
8931
8932 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
8933 static int
8934 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
8935                            __rte_unused struct rte_flow_tunnel *app_tunnel,
8936                            __rte_unused struct rte_flow_action **actions,
8937                            __rte_unused uint32_t *num_of_actions,
8938                            __rte_unused struct rte_flow_error *error)
8939 {
8940         return -ENOTSUP;
8941 }
8942
8943 static int
8944 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
8945                        __rte_unused struct rte_flow_tunnel *app_tunnel,
8946                        __rte_unused struct rte_flow_item **items,
8947                        __rte_unused uint32_t *num_of_items,
8948                        __rte_unused struct rte_flow_error *error)
8949 {
8950         return -ENOTSUP;
8951 }
8952
8953 static int
8954 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
8955                               __rte_unused struct rte_flow_item *pmd_items,
8956                               __rte_unused uint32_t num_items,
8957                               __rte_unused struct rte_flow_error *err)
8958 {
8959         return -ENOTSUP;
8960 }
8961
8962 static int
8963 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
8964                                 __rte_unused struct rte_flow_action *pmd_action,
8965                                 __rte_unused uint32_t num_actions,
8966                                 __rte_unused struct rte_flow_error *err)
8967 {
8968         return -ENOTSUP;
8969 }
8970
8971 static int
8972 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
8973                                   __rte_unused struct rte_mbuf *m,
8974                                   __rte_unused struct rte_flow_restore_info *i,
8975                                   __rte_unused struct rte_flow_error *err)
8976 {
8977         return -ENOTSUP;
8978 }
8979
8980 static int
8981 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
8982                              __rte_unused struct rte_flow *flow,
8983                              __rte_unused const struct rte_flow_attr *attr,
8984                              __rte_unused const struct rte_flow_action *actions,
8985                              __rte_unused uint32_t flow_idx,
8986                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
8987                              __rte_unused struct tunnel_default_miss_ctx *ctx,
8988                              __rte_unused struct rte_flow_error *error)
8989 {
8990         return -ENOTSUP;
8991 }
8992
8993 static struct mlx5_flow_tunnel *
8994 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
8995                     __rte_unused uint32_t id)
8996 {
8997         return NULL;
8998 }
8999
9000 static void
9001 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
9002                       __rte_unused struct mlx5_flow_tunnel *tunnel)
9003 {
9004 }
9005
9006 static uint32_t
9007 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
9008                                 __rte_unused const struct mlx5_flow_tunnel *t,
9009                                 __rte_unused uint32_t group,
9010                                 __rte_unused uint32_t *table,
9011                                 struct rte_flow_error *error)
9012 {
9013         return rte_flow_error_set(error, ENOTSUP,
9014                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9015                                   "tunnel offload requires DV support");
9016 }
9017
9018 void
9019 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
9020                         __rte_unused  uint16_t port_id)
9021 {
9022 }
9023 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
9024
9025 static void
9026 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
9027 {
9028         int ret;
9029         struct rte_flow_error error;
9030
9031         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9032                 char *item_name;
9033                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
9034                                     sizeof(item_name),
9035                                     (void *)(uintptr_t)item->type, &error);
9036                 if (ret > 0)
9037                         printf("%s ", item_name);
9038                 else
9039                         printf("%d\n", (int)item->type);
9040         }
9041         printf("END\n");
9042 }