drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11
  12 /* Verbs header. */
  13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  14 #ifdef PEDANTIC
  15 #pragma GCC diagnostic ignored "-Wpedantic"
  16 #endif
  17 #include <infiniband/verbs.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_common.h>
  23 #include <rte_ether.h>
  24 #include <rte_ethdev_driver.h>
  25 #include <rte_flow.h>
  26 #include <rte_flow_driver.h>
  27 #include <rte_malloc.h>
  28 #include <rte_ip.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_flow.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_prm.h"
  35 #include "mlx5_rxtx.h"
  36
  37 /* Dev ops structure defined in mlx5.c */
  38 extern const struct eth_dev_ops mlx5_dev_ops;
  39 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
  40
  41 /** Device flow drivers. */
  42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
  44 #endif
  45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  46
  47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  48
  49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  50         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  52         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  53 #endif
  54         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  55         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  56 };
  57
  58 enum mlx5_expansion {
  59         MLX5_EXPANSION_ROOT,
  60         MLX5_EXPANSION_ROOT_OUTER,
  61         MLX5_EXPANSION_ROOT_ETH_VLAN,
  62         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
  63         MLX5_EXPANSION_OUTER_ETH,
  64         MLX5_EXPANSION_OUTER_ETH_VLAN,
  65         MLX5_EXPANSION_OUTER_VLAN,
  66         MLX5_EXPANSION_OUTER_IPV4,
  67         MLX5_EXPANSION_OUTER_IPV4_UDP,
  68         MLX5_EXPANSION_OUTER_IPV4_TCP,
  69         MLX5_EXPANSION_OUTER_IPV6,
  70         MLX5_EXPANSION_OUTER_IPV6_UDP,
  71         MLX5_EXPANSION_OUTER_IPV6_TCP,
  72         MLX5_EXPANSION_VXLAN,
  73         MLX5_EXPANSION_VXLAN_GPE,
  74         MLX5_EXPANSION_GRE,
  75         MLX5_EXPANSION_MPLS,
  76         MLX5_EXPANSION_ETH,
  77         MLX5_EXPANSION_ETH_VLAN,
  78         MLX5_EXPANSION_VLAN,
  79         MLX5_EXPANSION_IPV4,
  80         MLX5_EXPANSION_IPV4_UDP,
  81         MLX5_EXPANSION_IPV4_TCP,
  82         MLX5_EXPANSION_IPV6,
  83         MLX5_EXPANSION_IPV6_UDP,
  84         MLX5_EXPANSION_IPV6_TCP,
  85 };
  86
  87 /** Supported expansion of items. */
  88 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
  89         [MLX5_EXPANSION_ROOT] = {
  90                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
  91                                                  MLX5_EXPANSION_IPV4,
  92                                                  MLX5_EXPANSION_IPV6),
  93                 .type = RTE_FLOW_ITEM_TYPE_END,
  94         },
  95         [MLX5_EXPANSION_ROOT_OUTER] = {
  96                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
  97                                                  MLX5_EXPANSION_OUTER_IPV4,
  98                                                  MLX5_EXPANSION_OUTER_IPV6),
  99                 .type = RTE_FLOW_ITEM_TYPE_END,
 100         },
 101         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 102                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 103                 .type = RTE_FLOW_ITEM_TYPE_END,
 104         },
 105         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 106                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
 107                 .type = RTE_FLOW_ITEM_TYPE_END,
 108         },
 109         [MLX5_EXPANSION_OUTER_ETH] = {
 110                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 111                                                  MLX5_EXPANSION_OUTER_IPV6,
 112                                                  MLX5_EXPANSION_MPLS),
 113                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 114                 .rss_types = 0,
 115         },
 116         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 117                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 118                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 119                 .rss_types = 0,
 120         },
 121         [MLX5_EXPANSION_OUTER_VLAN] = {
 122                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 123                                                  MLX5_EXPANSION_OUTER_IPV6),
 124                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 125         },
 126         [MLX5_EXPANSION_OUTER_IPV4] = {
 127                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 128                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 129                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 130                          MLX5_EXPANSION_GRE,
 131                          MLX5_EXPANSION_IPV4,
 132                          MLX5_EXPANSION_IPV6),
 133                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 134                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 135                         ETH_RSS_NONFRAG_IPV4_OTHER,
 136         },
 137         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 138                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 139                                                  MLX5_EXPANSION_VXLAN_GPE),
 140                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 141                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 142         },
 143         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 144                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 145                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 146         },
 147         [MLX5_EXPANSION_OUTER_IPV6] = {
 148                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 149                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 150                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 151                          MLX5_EXPANSION_IPV4,
 152                          MLX5_EXPANSION_IPV6),
 153                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 154                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 155                         ETH_RSS_NONFRAG_IPV6_OTHER,
 156         },
 157         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 158                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 159                                                  MLX5_EXPANSION_VXLAN_GPE),
 160                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 161                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 162         },
 163         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 164                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 165                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 166         },
 167         [MLX5_EXPANSION_VXLAN] = {
 168                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 169                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 170         },
 171         [MLX5_EXPANSION_VXLAN_GPE] = {
 172                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 173                                                  MLX5_EXPANSION_IPV4,
 174                                                  MLX5_EXPANSION_IPV6),
 175                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 176         },
 177         [MLX5_EXPANSION_GRE] = {
 178                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 179                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 180         },
 181         [MLX5_EXPANSION_MPLS] = {
 182                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 183                                                  MLX5_EXPANSION_IPV6),
 184                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 185         },
 186         [MLX5_EXPANSION_ETH] = {
 187                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 188                                                  MLX5_EXPANSION_IPV6),
 189                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 190         },
 191         [MLX5_EXPANSION_ETH_VLAN] = {
 192                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 193                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 194         },
 195         [MLX5_EXPANSION_VLAN] = {
 196                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 197                                                  MLX5_EXPANSION_IPV6),
 198                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 199         },
 200         [MLX5_EXPANSION_IPV4] = {
 201                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 202                                                  MLX5_EXPANSION_IPV4_TCP),
 203                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 204                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 205                         ETH_RSS_NONFRAG_IPV4_OTHER,
 206         },
 207         [MLX5_EXPANSION_IPV4_UDP] = {
 208                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 209                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 210         },
 211         [MLX5_EXPANSION_IPV4_TCP] = {
 212                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 213                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 214         },
 215         [MLX5_EXPANSION_IPV6] = {
 216                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 217                                                  MLX5_EXPANSION_IPV6_TCP),
 218                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 219                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 220                         ETH_RSS_NONFRAG_IPV6_OTHER,
 221         },
 222         [MLX5_EXPANSION_IPV6_UDP] = {
 223                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 224                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 225         },
 226         [MLX5_EXPANSION_IPV6_TCP] = {
 227                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 228                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 229         },
 230 };
 231
 232 static const struct rte_flow_ops mlx5_flow_ops = {
 233         .validate = mlx5_flow_validate,
 234         .create = mlx5_flow_create,
 235         .destroy = mlx5_flow_destroy,
 236         .flush = mlx5_flow_flush,
 237         .isolate = mlx5_flow_isolate,
 238         .query = mlx5_flow_query,
 239         .dev_dump = mlx5_flow_dev_dump,
 240 };
 241
 242 /* Convert FDIR request to Generic flow. */
 243 struct mlx5_fdir {
 244         struct rte_flow_attr attr;
 245         struct rte_flow_item items[4];
 246         struct rte_flow_item_eth l2;
 247         struct rte_flow_item_eth l2_mask;
 248         union {
 249                 struct rte_flow_item_ipv4 ipv4;
 250                 struct rte_flow_item_ipv6 ipv6;
 251         } l3;
 252         union {
 253                 struct rte_flow_item_ipv4 ipv4;
 254                 struct rte_flow_item_ipv6 ipv6;
 255         } l3_mask;
 256         union {
 257                 struct rte_flow_item_udp udp;
 258                 struct rte_flow_item_tcp tcp;
 259         } l4;
 260         union {
 261                 struct rte_flow_item_udp udp;
 262                 struct rte_flow_item_tcp tcp;
 263         } l4_mask;
 264         struct rte_flow_action actions[2];
 265         struct rte_flow_action_queue queue;
 266 };
 267
 268 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
 269 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
 270         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
 271 };
 272
 273 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
 274 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 275         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
 276         { 9, 10, 11 }, { 12, 13, 14 },
 277 };
 278
 279 /* Tunnel information. */
 280 struct mlx5_flow_tunnel_info {
 281         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 282         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 283 };
 284
 285 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 286         {
 287                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 288                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 289         },
 290         {
 291                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 292                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 293         },
 294         {
 295                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 296                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 297         },
 298         {
 299                 .tunnel = MLX5_FLOW_LAYER_GRE,
 300                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 301         },
 302         {
 303                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 304                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 305         },
 306         {
 307                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 308                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 309         },
 310         {
 311                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 312                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 313         },
 314         {
 315                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 316                 .ptype = RTE_PTYPE_TUNNEL_IP,
 317         },
 318         {
 319                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 320                 .ptype = RTE_PTYPE_TUNNEL_IP,
 321         },
 322         {
 323                 .tunnel = MLX5_FLOW_LAYER_GTP,
 324                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 325         },
 326 };
 327
 328 /**
 329  * Translate tag ID to register.
 330  *
 331  * @param[in] dev
 332  *   Pointer to the Ethernet device structure.
 333  * @param[in] feature
 334  *   The feature that request the register.
 335  * @param[in] id
 336  *   The request register ID.
 337  * @param[out] error
 338  *   Error description in case of any.
 339  *
 340  * @return
 341  *   The request register on success, a negative errno
 342  *   value otherwise and rte_errno is set.
 343  */
 344 enum modify_reg
 345 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 346                      enum mlx5_feature_name feature,
 347                      uint32_t id,
 348                      struct rte_flow_error *error)
 349 {
 350         struct mlx5_priv *priv = dev->data->dev_private;
 351         struct mlx5_dev_config *config = &priv->config;
 352         enum modify_reg start_reg;
 353
 354         switch (feature) {
 355         case MLX5_HAIRPIN_RX:
 356                 return REG_B;
 357         case MLX5_HAIRPIN_TX:
 358                 return REG_A;
 359         case MLX5_METADATA_RX:
 360                 switch (config->dv_xmeta_en) {
 361                 case MLX5_XMETA_MODE_LEGACY:
 362                         return REG_B;
 363                 case MLX5_XMETA_MODE_META16:
 364                         return REG_C_0;
 365                 case MLX5_XMETA_MODE_META32:
 366                         return REG_C_1;
 367                 }
 368                 break;
 369         case MLX5_METADATA_TX:
 370                 return REG_A;
 371         case MLX5_METADATA_FDB:
 372                 switch (config->dv_xmeta_en) {
 373                 case MLX5_XMETA_MODE_LEGACY:
 374                         return REG_NONE;
 375                 case MLX5_XMETA_MODE_META16:
 376                         return REG_C_0;
 377                 case MLX5_XMETA_MODE_META32:
 378                         return REG_C_1;
 379                 }
 380                 break;
 381         case MLX5_FLOW_MARK:
 382                 switch (config->dv_xmeta_en) {
 383                 case MLX5_XMETA_MODE_LEGACY:
 384                         return REG_NONE;
 385                 case MLX5_XMETA_MODE_META16:
 386                         return REG_C_1;
 387                 case MLX5_XMETA_MODE_META32:
 388                         return REG_C_0;
 389                 }
 390                 break;
 391         case MLX5_COPY_MARK:
 392         case MLX5_MTR_SFX:
 393                 /*
 394                  * Metadata COPY_MARK register using is in meter suffix sub
 395                  * flow while with meter. It's safe to share the same register.
 396                  */
 397                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 398         case MLX5_MTR_COLOR:
 399                 RTE_ASSERT(priv->mtr_color_reg != REG_NONE);
 400                 return priv->mtr_color_reg;
 401         case MLX5_APP_TAG:
 402                 /*
 403                  * If meter is enable, it will engage two registers for color
 404                  * match and flow match. If meter color match is not using the
 405                  * REG_C_2, need to skip the REG_C_x be used by meter color
 406                  * match.
 407                  * If meter is disable, free to use all available registers.
 408                  */
 409                 if (priv->mtr_color_reg != REG_NONE)
 410                         start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_3 :
 411                                     REG_C_4;
 412                 else
 413                         start_reg = REG_C_2;
 414                 if (id > (REG_C_7 - start_reg))
 415                         return rte_flow_error_set(error, EINVAL,
 416                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 417                                                   NULL, "invalid tag id");
 418                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE)
 419                         return rte_flow_error_set(error, ENOTSUP,
 420                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 421                                                   NULL, "unsupported tag id");
 422                 /*
 423                  * This case means meter is using the REG_C_x great than 2.
 424                  * Take care not to conflict with meter color REG_C_x.
 425                  * If the available index REG_C_y >= REG_C_x, skip the
 426                  * color register.
 427                  */
 428                 if (start_reg == REG_C_3 && config->flow_mreg_c
 429                     [id + REG_C_3 - REG_C_0] >= priv->mtr_color_reg) {
 430                         if (config->flow_mreg_c[id + 1 + REG_C_3 - REG_C_0] !=
 431                             REG_NONE)
 432                                 return config->flow_mreg_c
 433                                                 [id + 1 + REG_C_3 - REG_C_0];
 434                         return rte_flow_error_set(error, ENOTSUP,
 435                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 436                                                   NULL, "unsupported tag id");
 437                 }
 438                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 439         }
 440         assert(false);
 441         return rte_flow_error_set(error, EINVAL,
 442                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 443                                   NULL, "invalid feature name");
 444 }
 445
 446 /**
 447  * Check extensive flow metadata register support.
 448  *
 449  * @param dev
 450  *   Pointer to rte_eth_dev structure.
 451  *
 452  * @return
 453  *   True if device supports extensive flow metadata register, otherwise false.
 454  */
 455 bool
 456 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 457 {
 458         struct mlx5_priv *priv = dev->data->dev_private;
 459         struct mlx5_dev_config *config = &priv->config;
 460
 461         /*
 462          * Having available reg_c can be regarded inclusively as supporting
 463          * extensive flow metadata register, which could mean,
 464          * - metadata register copy action by modify header.
 465          * - 16 modify header actions is supported.
 466          * - reg_c's are preserved across different domain (FDB and NIC) on
 467          *   packet loopback by flow lookup miss.
 468          */
 469         return config->flow_mreg_c[2] != REG_NONE;
 470 }
 471
 472 /**
 473  * Discover the maximum number of priority available.
 474  *
 475  * @param[in] dev
 476  *   Pointer to the Ethernet device structure.
 477  *
 478  * @return
 479  *   number of supported flow priority on success, a negative errno
 480  *   value otherwise and rte_errno is set.
 481  */
 482 int
 483 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 484 {
 485         struct mlx5_priv *priv = dev->data->dev_private;
 486         struct {
 487                 struct ibv_flow_attr attr;
 488                 struct ibv_flow_spec_eth eth;
 489                 struct ibv_flow_spec_action_drop drop;
 490         } flow_attr = {
 491                 .attr = {
 492                         .num_of_specs = 2,
 493                         .port = (uint8_t)priv->ibv_port,
 494                 },
 495                 .eth = {
 496                         .type = IBV_FLOW_SPEC_ETH,
 497                         .size = sizeof(struct ibv_flow_spec_eth),
 498                 },
 499                 .drop = {
 500                         .size = sizeof(struct ibv_flow_spec_action_drop),
 501                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 502                 },
 503         };
 504         struct ibv_flow *flow;
 505         struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 506         uint16_t vprio[] = { 8, 16 };
 507         int i;
 508         int priority = 0;
 509
 510         if (!drop) {
 511                 rte_errno = ENOTSUP;
 512                 return -rte_errno;
 513         }
 514         for (i = 0; i != RTE_DIM(vprio); i++) {
 515                 flow_attr.attr.priority = vprio[i] - 1;
 516                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 517                 if (!flow)
 518                         break;
 519                 claim_zero(mlx5_glue->destroy_flow(flow));
 520                 priority = vprio[i];
 521         }
 522         mlx5_hrxq_drop_release(dev);
 523         switch (priority) {
 524         case 8:
 525                 priority = RTE_DIM(priority_map_3);
 526                 break;
 527         case 16:
 528                 priority = RTE_DIM(priority_map_5);
 529                 break;
 530         default:
 531                 rte_errno = ENOTSUP;
 532                 DRV_LOG(ERR,
 533                         "port %u verbs maximum priority: %d expected 8/16",
 534                         dev->data->port_id, priority);
 535                 return -rte_errno;
 536         }
 537         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 538                 dev->data->port_id, priority);
 539         return priority;
 540 }
 541
 542 /**
 543  * Adjust flow priority based on the highest layer and the request priority.
 544  *
 545  * @param[in] dev
 546  *   Pointer to the Ethernet device structure.
 547  * @param[in] priority
 548  *   The rule base priority.
 549  * @param[in] subpriority
 550  *   The priority based on the items.
 551  *
 552  * @return
 553  *   The new priority.
 554  */
 555 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 556                                    uint32_t subpriority)
 557 {
 558         uint32_t res = 0;
 559         struct mlx5_priv *priv = dev->data->dev_private;
 560
 561         switch (priv->config.flow_prio) {
 562         case RTE_DIM(priority_map_3):
 563                 res = priority_map_3[priority][subpriority];
 564                 break;
 565         case RTE_DIM(priority_map_5):
 566                 res = priority_map_5[priority][subpriority];
 567                 break;
 568         }
 569         return  res;
 570 }
 571
 572 /**
 573  * Verify the @p item specifications (spec, last, mask) are compatible with the
 574  * NIC capabilities.
 575  *
 576  * @param[in] item
 577  *   Item specification.
 578  * @param[in] mask
 579  *   @p item->mask or flow default bit-masks.
 580  * @param[in] nic_mask
 581  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 582  * @param[in] size
 583  *   Bit-masks size in bytes.
 584  * @param[out] error
 585  *   Pointer to error structure.
 586  *
 587  * @return
 588  *   0 on success, a negative errno value otherwise and rte_errno is set.
 589  */
 590 int
 591 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 592                           const uint8_t *mask,
 593                           const uint8_t *nic_mask,
 594                           unsigned int size,
 595                           struct rte_flow_error *error)
 596 {
 597         unsigned int i;
 598
 599         assert(nic_mask);
 600         for (i = 0; i < size; ++i)
 601                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 602                         return rte_flow_error_set(error, ENOTSUP,
 603                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 604                                                   item,
 605                                                   "mask enables non supported"
 606                                                   " bits");
 607         if (!item->spec && (item->mask || item->last))
 608                 return rte_flow_error_set(error, EINVAL,
 609                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 610                                           "mask/last without a spec is not"
 611                                           " supported");
 612         if (item->spec && item->last) {
 613                 uint8_t spec[size];
 614                 uint8_t last[size];
 615                 unsigned int i;
 616                 int ret;
 617
 618                 for (i = 0; i < size; ++i) {
 619                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 620                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 621                 }
 622                 ret = memcmp(spec, last, size);
 623                 if (ret != 0)
 624                         return rte_flow_error_set(error, EINVAL,
 625                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 626                                                   item,
 627                                                   "range is not valid");
 628         }
 629         return 0;
 630 }
 631
 632 /**
 633  * Adjust the hash fields according to the @p flow information.
 634  *
 635  * @param[in] dev_flow.
 636  *   Pointer to the mlx5_flow.
 637  * @param[in] tunnel
 638  *   1 when the hash field is for a tunnel item.
 639  * @param[in] layer_types
 640  *   ETH_RSS_* types.
 641  * @param[in] hash_fields
 642  *   Item hash fields.
 643  *
 644  * @return
 645  *   The hash fields that should be used.
 646  */
 647 uint64_t
 648 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
 649                             int tunnel __rte_unused, uint64_t layer_types,
 650                             uint64_t hash_fields)
 651 {
 652         struct rte_flow *flow = dev_flow->flow;
 653 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 654         int rss_request_inner = flow->rss.level >= 2;
 655
 656         /* Check RSS hash level for tunnel. */
 657         if (tunnel && rss_request_inner)
 658                 hash_fields |= IBV_RX_HASH_INNER;
 659         else if (tunnel || rss_request_inner)
 660                 return 0;
 661 #endif
 662         /* Check if requested layer matches RSS hash fields. */
 663         if (!(flow->rss.types & layer_types))
 664                 return 0;
 665         return hash_fields;
 666 }
 667
 668 /**
 669  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 670  * if several tunnel rules are used on this queue, the tunnel ptype will be
 671  * cleared.
 672  *
 673  * @param rxq_ctrl
 674  *   Rx queue to update.
 675  */
 676 static void
 677 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 678 {
 679         unsigned int i;
 680         uint32_t tunnel_ptype = 0;
 681
 682         /* Look up for the ptype to use. */
 683         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 684                 if (!rxq_ctrl->flow_tunnels_n[i])
 685                         continue;
 686                 if (!tunnel_ptype) {
 687                         tunnel_ptype = tunnels_info[i].ptype;
 688                 } else {
 689                         tunnel_ptype = 0;
 690                         break;
 691                 }
 692         }
 693         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 694 }
 695
 696 /**
 697  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 698  * flow.
 699  *
 700  * @param[in] dev
 701  *   Pointer to the Ethernet device structure.
 702  * @param[in] dev_flow
 703  *   Pointer to device flow structure.
 704  */
 705 static void
 706 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 707 {
 708         struct mlx5_priv *priv = dev->data->dev_private;
 709         struct rte_flow *flow = dev_flow->flow;
 710         const int mark = !!(dev_flow->actions &
 711                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 712         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 713         unsigned int i;
 714
 715         for (i = 0; i != flow->rss.queue_num; ++i) {
 716                 int idx = (*flow->rss.queue)[i];
 717                 struct mlx5_rxq_ctrl *rxq_ctrl =
 718                         container_of((*priv->rxqs)[idx],
 719                                      struct mlx5_rxq_ctrl, rxq);
 720
 721                 /*
 722                  * To support metadata register copy on Tx loopback,
 723                  * this must be always enabled (metadata may arive
 724                  * from other port - not from local flows only.
 725                  */
 726                 if (priv->config.dv_flow_en &&
 727                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
 728                     mlx5_flow_ext_mreg_supported(dev)) {
 729                         rxq_ctrl->rxq.mark = 1;
 730                         rxq_ctrl->flow_mark_n = 1;
 731                 } else if (mark) {
 732                         rxq_ctrl->rxq.mark = 1;
 733                         rxq_ctrl->flow_mark_n++;
 734                 }
 735                 if (tunnel) {
 736                         unsigned int j;
 737
 738                         /* Increase the counter matching the flow. */
 739                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 740                                 if ((tunnels_info[j].tunnel &
 741                                      dev_flow->layers) ==
 742                                     tunnels_info[j].tunnel) {
 743                                         rxq_ctrl->flow_tunnels_n[j]++;
 744                                         break;
 745                                 }
 746                         }
 747                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 748                 }
 749         }
 750 }
 751
 752 /**
 753  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
 754  *
 755  * @param[in] dev
 756  *   Pointer to the Ethernet device structure.
 757  * @param[in] flow
 758  *   Pointer to flow structure.
 759  */
 760 static void
 761 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 762 {
 763         struct mlx5_flow *dev_flow;
 764
 765         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 766                 flow_drv_rxq_flags_set(dev, dev_flow);
 767 }
 768
 769 /**
 770  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 771  * device flow if no other flow uses it with the same kind of request.
 772  *
 773  * @param dev
 774  *   Pointer to Ethernet device.
 775  * @param[in] dev_flow
 776  *   Pointer to the device flow.
 777  */
 778 static void
 779 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 780 {
 781         struct mlx5_priv *priv = dev->data->dev_private;
 782         struct rte_flow *flow = dev_flow->flow;
 783         const int mark = !!(dev_flow->actions &
 784                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 785         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 786         unsigned int i;
 787
 788         assert(dev->data->dev_started);
 789         for (i = 0; i != flow->rss.queue_num; ++i) {
 790                 int idx = (*flow->rss.queue)[i];
 791                 struct mlx5_rxq_ctrl *rxq_ctrl =
 792                         container_of((*priv->rxqs)[idx],
 793                                      struct mlx5_rxq_ctrl, rxq);
 794
 795                 if (priv->config.dv_flow_en &&
 796                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
 797                     mlx5_flow_ext_mreg_supported(dev)) {
 798                         rxq_ctrl->rxq.mark = 1;
 799                         rxq_ctrl->flow_mark_n = 1;
 800                 } else if (mark) {
 801                         rxq_ctrl->flow_mark_n--;
 802                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 803                 }
 804                 if (tunnel) {
 805                         unsigned int j;
 806
 807                         /* Decrease the counter matching the flow. */
 808                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 809                                 if ((tunnels_info[j].tunnel &
 810                                      dev_flow->layers) ==
 811                                     tunnels_info[j].tunnel) {
 812                                         rxq_ctrl->flow_tunnels_n[j]--;
 813                                         break;
 814                                 }
 815                         }
 816                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 817                 }
 818         }
 819 }
 820
 821 /**
 822  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 823  * @p flow if no other flow uses it with the same kind of request.
 824  *
 825  * @param dev
 826  *   Pointer to Ethernet device.
 827  * @param[in] flow
 828  *   Pointer to the flow.
 829  */
 830 static void
 831 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 832 {
 833         struct mlx5_flow *dev_flow;
 834
 835         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 836                 flow_drv_rxq_flags_trim(dev, dev_flow);
 837 }
 838
 839 /**
 840  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
 841  *
 842  * @param dev
 843  *   Pointer to Ethernet device.
 844  */
 845 static void
 846 flow_rxq_flags_clear(struct rte_eth_dev *dev)
 847 {
 848         struct mlx5_priv *priv = dev->data->dev_private;
 849         unsigned int i;
 850
 851         for (i = 0; i != priv->rxqs_n; ++i) {
 852                 struct mlx5_rxq_ctrl *rxq_ctrl;
 853                 unsigned int j;
 854
 855                 if (!(*priv->rxqs)[i])
 856                         continue;
 857                 rxq_ctrl = container_of((*priv->rxqs)[i],
 858                                         struct mlx5_rxq_ctrl, rxq);
 859                 rxq_ctrl->flow_mark_n = 0;
 860                 rxq_ctrl->rxq.mark = 0;
 861                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
 862                         rxq_ctrl->flow_tunnels_n[j] = 0;
 863                 rxq_ctrl->rxq.tunnel = 0;
 864         }
 865 }
 866
 867 /*
 868  * return a pointer to the desired action in the list of actions.
 869  *
 870  * @param[in] actions
 871  *   The list of actions to search the action in.
 872  * @param[in] action
 873  *   The action to find.
 874  *
 875  * @return
 876  *   Pointer to the action in the list, if found. NULL otherwise.
 877  */
 878 const struct rte_flow_action *
 879 mlx5_flow_find_action(const struct rte_flow_action *actions,
 880                       enum rte_flow_action_type action)
 881 {
 882         if (actions == NULL)
 883                 return NULL;
 884         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
 885                 if (actions->type == action)
 886                         return actions;
 887         return NULL;
 888 }
 889
 890 /*
 891  * Validate the flag action.
 892  *
 893  * @param[in] action_flags
 894  *   Bit-fields that holds the actions detected until now.
 895  * @param[in] attr
 896  *   Attributes of flow that includes this action.
 897  * @param[out] error
 898  *   Pointer to error structure.
 899  *
 900  * @return
 901  *   0 on success, a negative errno value otherwise and rte_errno is set.
 902  */
 903 int
 904 mlx5_flow_validate_action_flag(uint64_t action_flags,
 905                                const struct rte_flow_attr *attr,
 906                                struct rte_flow_error *error)
 907 {
 908         if (action_flags & MLX5_FLOW_ACTION_MARK)
 909                 return rte_flow_error_set(error, EINVAL,
 910                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 911                                           "can't mark and flag in same flow");
 912         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 913                 return rte_flow_error_set(error, EINVAL,
 914                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 915                                           "can't have 2 flag"
 916                                           " actions in same flow");
 917         if (attr->egress)
 918                 return rte_flow_error_set(error, ENOTSUP,
 919                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 920                                           "flag action not supported for "
 921                                           "egress");
 922         return 0;
 923 }
 924
 925 /*
 926  * Validate the mark action.
 927  *
 928  * @param[in] action
 929  *   Pointer to the queue action.
 930  * @param[in] action_flags
 931  *   Bit-fields that holds the actions detected until now.
 932  * @param[in] attr
 933  *   Attributes of flow that includes this action.
 934  * @param[out] error
 935  *   Pointer to error structure.
 936  *
 937  * @return
 938  *   0 on success, a negative errno value otherwise and rte_errno is set.
 939  */
 940 int
 941 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
 942                                uint64_t action_flags,
 943                                const struct rte_flow_attr *attr,
 944                                struct rte_flow_error *error)
 945 {
 946         const struct rte_flow_action_mark *mark = action->conf;
 947
 948         if (!mark)
 949                 return rte_flow_error_set(error, EINVAL,
 950                                           RTE_FLOW_ERROR_TYPE_ACTION,
 951                                           action,
 952                                           "configuration cannot be null");
 953         if (mark->id >= MLX5_FLOW_MARK_MAX)
 954                 return rte_flow_error_set(error, EINVAL,
 955                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 956                                           &mark->id,
 957                                           "mark id must in 0 <= id < "
 958                                           RTE_STR(MLX5_FLOW_MARK_MAX));
 959         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 960                 return rte_flow_error_set(error, EINVAL,
 961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 962                                           "can't flag and mark in same flow");
 963         if (action_flags & MLX5_FLOW_ACTION_MARK)
 964                 return rte_flow_error_set(error, EINVAL,
 965                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 966                                           "can't have 2 mark actions in same"
 967                                           " flow");
 968         if (attr->egress)
 969                 return rte_flow_error_set(error, ENOTSUP,
 970                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 971                                           "mark action not supported for "
 972                                           "egress");
 973         return 0;
 974 }
 975
 976 /*
 977  * Validate the drop action.
 978  *
 979  * @param[in] action_flags
 980  *   Bit-fields that holds the actions detected until now.
 981  * @param[in] attr
 982  *   Attributes of flow that includes this action.
 983  * @param[out] error
 984  *   Pointer to error structure.
 985  *
 986  * @return
 987  *   0 on success, a negative errno value otherwise and rte_errno is set.
 988  */
 989 int
 990 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
 991                                const struct rte_flow_attr *attr,
 992                                struct rte_flow_error *error)
 993 {
 994         if (attr->egress)
 995                 return rte_flow_error_set(error, ENOTSUP,
 996                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 997                                           "drop action not supported for "
 998                                           "egress");
 999         return 0;
1000 }
1001
1002 /*
1003  * Validate the queue action.
1004  *
1005  * @param[in] action
1006  *   Pointer to the queue action.
1007  * @param[in] action_flags
1008  *   Bit-fields that holds the actions detected until now.
1009  * @param[in] dev
1010  *   Pointer to the Ethernet device structure.
1011  * @param[in] attr
1012  *   Attributes of flow that includes this action.
1013  * @param[out] error
1014  *   Pointer to error structure.
1015  *
1016  * @return
1017  *   0 on success, a negative errno value otherwise and rte_errno is set.
1018  */
1019 int
1020 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1021                                 uint64_t action_flags,
1022                                 struct rte_eth_dev *dev,
1023                                 const struct rte_flow_attr *attr,
1024                                 struct rte_flow_error *error)
1025 {
1026         struct mlx5_priv *priv = dev->data->dev_private;
1027         const struct rte_flow_action_queue *queue = action->conf;
1028
1029         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1030                 return rte_flow_error_set(error, EINVAL,
1031                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1032                                           "can't have 2 fate actions in"
1033                                           " same flow");
1034         if (!priv->rxqs_n)
1035                 return rte_flow_error_set(error, EINVAL,
1036                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1037                                           NULL, "No Rx queues configured");
1038         if (queue->index >= priv->rxqs_n)
1039                 return rte_flow_error_set(error, EINVAL,
1040                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1041                                           &queue->index,
1042                                           "queue index out of range");
1043         if (!(*priv->rxqs)[queue->index])
1044                 return rte_flow_error_set(error, EINVAL,
1045                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1046                                           &queue->index,
1047                                           "queue is not configured");
1048         if (attr->egress)
1049                 return rte_flow_error_set(error, ENOTSUP,
1050                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1051                                           "queue action not supported for "
1052                                           "egress");
1053         return 0;
1054 }
1055
1056 /*
1057  * Validate the rss action.
1058  *
1059  * @param[in] action
1060  *   Pointer to the queue action.
1061  * @param[in] action_flags
1062  *   Bit-fields that holds the actions detected until now.
1063  * @param[in] dev
1064  *   Pointer to the Ethernet device structure.
1065  * @param[in] attr
1066  *   Attributes of flow that includes this action.
1067  * @param[in] item_flags
1068  *   Items that were detected.
1069  * @param[out] error
1070  *   Pointer to error structure.
1071  *
1072  * @return
1073  *   0 on success, a negative errno value otherwise and rte_errno is set.
1074  */
1075 int
1076 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1077                               uint64_t action_flags,
1078                               struct rte_eth_dev *dev,
1079                               const struct rte_flow_attr *attr,
1080                               uint64_t item_flags,
1081                               struct rte_flow_error *error)
1082 {
1083         struct mlx5_priv *priv = dev->data->dev_private;
1084         const struct rte_flow_action_rss *rss = action->conf;
1085         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1086         unsigned int i;
1087
1088         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1089                 return rte_flow_error_set(error, EINVAL,
1090                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1091                                           "can't have 2 fate actions"
1092                                           " in same flow");
1093         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1094             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1095                 return rte_flow_error_set(error, ENOTSUP,
1096                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1097                                           &rss->func,
1098                                           "RSS hash function not supported");
1099 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1100         if (rss->level > 2)
1101 #else
1102         if (rss->level > 1)
1103 #endif
1104                 return rte_flow_error_set(error, ENOTSUP,
1105                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1106                                           &rss->level,
1107                                           "tunnel RSS is not supported");
1108         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1109         if (rss->key_len == 0 && rss->key != NULL)
1110                 return rte_flow_error_set(error, ENOTSUP,
1111                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1112                                           &rss->key_len,
1113                                           "RSS hash key length 0");
1114         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1115                 return rte_flow_error_set(error, ENOTSUP,
1116                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1117                                           &rss->key_len,
1118                                           "RSS hash key too small");
1119         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1120                 return rte_flow_error_set(error, ENOTSUP,
1121                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1122                                           &rss->key_len,
1123                                           "RSS hash key too large");
1124         if (rss->queue_num > priv->config.ind_table_max_size)
1125                 return rte_flow_error_set(error, ENOTSUP,
1126                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1127                                           &rss->queue_num,
1128                                           "number of queues too large");
1129         if (rss->types & MLX5_RSS_HF_MASK)
1130                 return rte_flow_error_set(error, ENOTSUP,
1131                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1132                                           &rss->types,
1133                                           "some RSS protocols are not"
1134                                           " supported");
1135         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1136             !(rss->types & ETH_RSS_IP))
1137                 return rte_flow_error_set(error, EINVAL,
1138                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1139                                           "L3 partial RSS requested but L3 RSS"
1140                                           " type not specified");
1141         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1142             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1143                 return rte_flow_error_set(error, EINVAL,
1144                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1145                                           "L4 partial RSS requested but L4 RSS"
1146                                           " type not specified");
1147         if (!priv->rxqs_n)
1148                 return rte_flow_error_set(error, EINVAL,
1149                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1150                                           NULL, "No Rx queues configured");
1151         if (!rss->queue_num)
1152                 return rte_flow_error_set(error, EINVAL,
1153                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1154                                           NULL, "No queues configured");
1155         for (i = 0; i != rss->queue_num; ++i) {
1156                 if (rss->queue[i] >= priv->rxqs_n)
1157                         return rte_flow_error_set
1158                                 (error, EINVAL,
1159                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1160                                  &rss->queue[i], "queue index out of range");
1161                 if (!(*priv->rxqs)[rss->queue[i]])
1162                         return rte_flow_error_set
1163                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1164                                  &rss->queue[i], "queue is not configured");
1165         }
1166         if (attr->egress)
1167                 return rte_flow_error_set(error, ENOTSUP,
1168                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1169                                           "rss action not supported for "
1170                                           "egress");
1171         if (rss->level > 1 &&  !tunnel)
1172                 return rte_flow_error_set(error, EINVAL,
1173                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1174                                           "inner RSS is not supported for "
1175                                           "non-tunnel flows");
1176         return 0;
1177 }
1178
1179 /*
1180  * Validate the count action.
1181  *
1182  * @param[in] dev
1183  *   Pointer to the Ethernet device structure.
1184  * @param[in] attr
1185  *   Attributes of flow that includes this action.
1186  * @param[out] error
1187  *   Pointer to error structure.
1188  *
1189  * @return
1190  *   0 on success, a negative errno value otherwise and rte_errno is set.
1191  */
1192 int
1193 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1194                                 const struct rte_flow_attr *attr,
1195                                 struct rte_flow_error *error)
1196 {
1197         if (attr->egress)
1198                 return rte_flow_error_set(error, ENOTSUP,
1199                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1200                                           "count action not supported for "
1201                                           "egress");
1202         return 0;
1203 }
1204
1205 /**
1206  * Verify the @p attributes will be correctly understood by the NIC and store
1207  * them in the @p flow if everything is correct.
1208  *
1209  * @param[in] dev
1210  *   Pointer to the Ethernet device structure.
1211  * @param[in] attributes
1212  *   Pointer to flow attributes
1213  * @param[out] error
1214  *   Pointer to error structure.
1215  *
1216  * @return
1217  *   0 on success, a negative errno value otherwise and rte_errno is set.
1218  */
1219 int
1220 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1221                               const struct rte_flow_attr *attributes,
1222                               struct rte_flow_error *error)
1223 {
1224         struct mlx5_priv *priv = dev->data->dev_private;
1225         uint32_t priority_max = priv->config.flow_prio - 1;
1226
1227         if (attributes->group)
1228                 return rte_flow_error_set(error, ENOTSUP,
1229                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1230                                           NULL, "groups is not supported");
1231         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1232             attributes->priority >= priority_max)
1233                 return rte_flow_error_set(error, ENOTSUP,
1234                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1235                                           NULL, "priority out of range");
1236         if (attributes->egress)
1237                 return rte_flow_error_set(error, ENOTSUP,
1238                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1239                                           "egress is not supported");
1240         if (attributes->transfer && !priv->config.dv_esw_en)
1241                 return rte_flow_error_set(error, ENOTSUP,
1242                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1243                                           NULL, "transfer is not supported");
1244         if (!attributes->ingress)
1245                 return rte_flow_error_set(error, EINVAL,
1246                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1247                                           NULL,
1248                                           "ingress attribute is mandatory");
1249         return 0;
1250 }
1251
1252 /**
1253  * Validate ICMP6 item.
1254  *
1255  * @param[in] item
1256  *   Item specification.
1257  * @param[in] item_flags
1258  *   Bit-fields that holds the items detected until now.
1259  * @param[out] error
1260  *   Pointer to error structure.
1261  *
1262  * @return
1263  *   0 on success, a negative errno value otherwise and rte_errno is set.
1264  */
1265 int
1266 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1267                                uint64_t item_flags,
1268                                uint8_t target_protocol,
1269                                struct rte_flow_error *error)
1270 {
1271         const struct rte_flow_item_icmp6 *mask = item->mask;
1272         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1273         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1274                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1275         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1276                                       MLX5_FLOW_LAYER_OUTER_L4;
1277         int ret;
1278
1279         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1280                 return rte_flow_error_set(error, EINVAL,
1281                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1282                                           "protocol filtering not compatible"
1283                                           " with ICMP6 layer");
1284         if (!(item_flags & l3m))
1285                 return rte_flow_error_set(error, EINVAL,
1286                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1287                                           "IPv6 is mandatory to filter on"
1288                                           " ICMP6");
1289         if (item_flags & l4m)
1290                 return rte_flow_error_set(error, EINVAL,
1291                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1292                                           "multiple L4 layers not supported");
1293         if (!mask)
1294                 mask = &rte_flow_item_icmp6_mask;
1295         ret = mlx5_flow_item_acceptable
1296                 (item, (const uint8_t *)mask,
1297                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1298                  sizeof(struct rte_flow_item_icmp6), error);
1299         if (ret < 0)
1300                 return ret;
1301         return 0;
1302 }
1303
1304 /**
1305  * Validate ICMP item.
1306  *
1307  * @param[in] item
1308  *   Item specification.
1309  * @param[in] item_flags
1310  *   Bit-fields that holds the items detected until now.
1311  * @param[out] error
1312  *   Pointer to error structure.
1313  *
1314  * @return
1315  *   0 on success, a negative errno value otherwise and rte_errno is set.
1316  */
1317 int
1318 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1319                              uint64_t item_flags,
1320                              uint8_t target_protocol,
1321                              struct rte_flow_error *error)
1322 {
1323         const struct rte_flow_item_icmp *mask = item->mask;
1324         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1325         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1326                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1327         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1328                                       MLX5_FLOW_LAYER_OUTER_L4;
1329         int ret;
1330
1331         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1332                 return rte_flow_error_set(error, EINVAL,
1333                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1334                                           "protocol filtering not compatible"
1335                                           " with ICMP layer");
1336         if (!(item_flags & l3m))
1337                 return rte_flow_error_set(error, EINVAL,
1338                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1339                                           "IPv4 is mandatory to filter"
1340                                           " on ICMP");
1341         if (item_flags & l4m)
1342                 return rte_flow_error_set(error, EINVAL,
1343                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1344                                           "multiple L4 layers not supported");
1345         if (!mask)
1346                 mask = &rte_flow_item_icmp_mask;
1347         ret = mlx5_flow_item_acceptable
1348                 (item, (const uint8_t *)mask,
1349                  (const uint8_t *)&rte_flow_item_icmp_mask,
1350                  sizeof(struct rte_flow_item_icmp), error);
1351         if (ret < 0)
1352                 return ret;
1353         return 0;
1354 }
1355
1356 /**
1357  * Validate Ethernet item.
1358  *
1359  * @param[in] item
1360  *   Item specification.
1361  * @param[in] item_flags
1362  *   Bit-fields that holds the items detected until now.
1363  * @param[out] error
1364  *   Pointer to error structure.
1365  *
1366  * @return
1367  *   0 on success, a negative errno value otherwise and rte_errno is set.
1368  */
1369 int
1370 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1371                             uint64_t item_flags,
1372                             struct rte_flow_error *error)
1373 {
1374         const struct rte_flow_item_eth *mask = item->mask;
1375         const struct rte_flow_item_eth nic_mask = {
1376                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1377                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1378                 .type = RTE_BE16(0xffff),
1379         };
1380         int ret;
1381         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1382         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1383                                        MLX5_FLOW_LAYER_OUTER_L2;
1384
1385         if (item_flags & ethm)
1386                 return rte_flow_error_set(error, ENOTSUP,
1387                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1388                                           "multiple L2 layers not supported");
1389         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1390             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1391                 return rte_flow_error_set(error, EINVAL,
1392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1393                                           "L2 layer should not follow "
1394                                           "L3 layers");
1395         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1396             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1397                 return rte_flow_error_set(error, EINVAL,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "L2 layer should not follow VLAN");
1400         if (!mask)
1401                 mask = &rte_flow_item_eth_mask;
1402         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1403                                         (const uint8_t *)&nic_mask,
1404                                         sizeof(struct rte_flow_item_eth),
1405                                         error);
1406         return ret;
1407 }
1408
1409 /**
1410  * Validate VLAN item.
1411  *
1412  * @param[in] item
1413  *   Item specification.
1414  * @param[in] item_flags
1415  *   Bit-fields that holds the items detected until now.
1416  * @param[in] dev
1417  *   Ethernet device flow is being created on.
1418  * @param[out] error
1419  *   Pointer to error structure.
1420  *
1421  * @return
1422  *   0 on success, a negative errno value otherwise and rte_errno is set.
1423  */
1424 int
1425 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1426                              uint64_t item_flags,
1427                              struct rte_eth_dev *dev,
1428                              struct rte_flow_error *error)
1429 {
1430         const struct rte_flow_item_vlan *spec = item->spec;
1431         const struct rte_flow_item_vlan *mask = item->mask;
1432         const struct rte_flow_item_vlan nic_mask = {
1433                 .tci = RTE_BE16(UINT16_MAX),
1434                 .inner_type = RTE_BE16(UINT16_MAX),
1435         };
1436         uint16_t vlan_tag = 0;
1437         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1438         int ret;
1439         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1440                                         MLX5_FLOW_LAYER_INNER_L4) :
1441                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1442                                         MLX5_FLOW_LAYER_OUTER_L4);
1443         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1444                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1445
1446         if (item_flags & vlanm)
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1449                                           "multiple VLAN layers not supported");
1450         else if ((item_flags & l34m) != 0)
1451                 return rte_flow_error_set(error, EINVAL,
1452                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1453                                           "VLAN cannot follow L3/L4 layer");
1454         if (!mask)
1455                 mask = &rte_flow_item_vlan_mask;
1456         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1457                                         (const uint8_t *)&nic_mask,
1458                                         sizeof(struct rte_flow_item_vlan),
1459                                         error);
1460         if (ret)
1461                 return ret;
1462         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1463                 struct mlx5_priv *priv = dev->data->dev_private;
1464
1465                 if (priv->vmwa_context) {
1466                         /*
1467                          * Non-NULL context means we have a virtual machine
1468                          * and SR-IOV enabled, we have to create VLAN interface
1469                          * to make hypervisor to setup E-Switch vport
1470                          * context correctly. We avoid creating the multiple
1471                          * VLAN interfaces, so we cannot support VLAN tag mask.
1472                          */
1473                         return rte_flow_error_set(error, EINVAL,
1474                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1475                                                   item,
1476                                                   "VLAN tag mask is not"
1477                                                   " supported in virtual"
1478                                                   " environment");
1479                 }
1480         }
1481         if (spec) {
1482                 vlan_tag = spec->tci;
1483                 vlan_tag &= mask->tci;
1484         }
1485         /*
1486          * From verbs perspective an empty VLAN is equivalent
1487          * to a packet without VLAN layer.
1488          */
1489         if (!vlan_tag)
1490                 return rte_flow_error_set(error, EINVAL,
1491                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1492                                           item->spec,
1493                                           "VLAN cannot be empty");
1494         return 0;
1495 }
1496
1497 /**
1498  * Validate IPV4 item.
1499  *
1500  * @param[in] item
1501  *   Item specification.
1502  * @param[in] item_flags
1503  *   Bit-fields that holds the items detected until now.
1504  * @param[in] acc_mask
1505  *   Acceptable mask, if NULL default internal default mask
1506  *   will be used to check whether item fields are supported.
1507  * @param[out] error
1508  *   Pointer to error structure.
1509  *
1510  * @return
1511  *   0 on success, a negative errno value otherwise and rte_errno is set.
1512  */
1513 int
1514 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1515                              uint64_t item_flags,
1516                              uint64_t last_item,
1517                              uint16_t ether_type,
1518                              const struct rte_flow_item_ipv4 *acc_mask,
1519                              struct rte_flow_error *error)
1520 {
1521         const struct rte_flow_item_ipv4 *mask = item->mask;
1522         const struct rte_flow_item_ipv4 *spec = item->spec;
1523         const struct rte_flow_item_ipv4 nic_mask = {
1524                 .hdr = {
1525                         .src_addr = RTE_BE32(0xffffffff),
1526                         .dst_addr = RTE_BE32(0xffffffff),
1527                         .type_of_service = 0xff,
1528                         .next_proto_id = 0xff,
1529                 },
1530         };
1531         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1532         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1533                                       MLX5_FLOW_LAYER_OUTER_L3;
1534         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1535                                       MLX5_FLOW_LAYER_OUTER_L4;
1536         int ret;
1537         uint8_t next_proto = 0xFF;
1538         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1539                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1540                                   MLX5_FLOW_LAYER_INNER_VLAN);
1541
1542         if ((last_item & l2_vlan) && ether_type &&
1543             ether_type != RTE_ETHER_TYPE_IPV4)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1546                                           "IPv4 cannot follow L2/VLAN layer "
1547                                           "which ether type is not IPv4");
1548         if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1549                 if (mask && spec)
1550                         next_proto = mask->hdr.next_proto_id &
1551                                      spec->hdr.next_proto_id;
1552                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1553                         return rte_flow_error_set(error, EINVAL,
1554                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1555                                                   item,
1556                                                   "multiple tunnel "
1557                                                   "not supported");
1558         }
1559         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1560                 return rte_flow_error_set(error, EINVAL,
1561                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1562                                           "wrong tunnel type - IPv6 specified "
1563                                           "but IPv4 item provided");
1564         if (item_flags & l3m)
1565                 return rte_flow_error_set(error, ENOTSUP,
1566                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1567                                           "multiple L3 layers not supported");
1568         else if (item_flags & l4m)
1569                 return rte_flow_error_set(error, EINVAL,
1570                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1571                                           "L3 cannot follow an L4 layer.");
1572         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1573                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1574                 return rte_flow_error_set(error, EINVAL,
1575                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1576                                           "L3 cannot follow an NVGRE layer.");
1577         if (!mask)
1578                 mask = &rte_flow_item_ipv4_mask;
1579         else if (mask->hdr.next_proto_id != 0 &&
1580                  mask->hdr.next_proto_id != 0xff)
1581                 return rte_flow_error_set(error, EINVAL,
1582                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1583                                           "partial mask is not supported"
1584                                           " for protocol");
1585         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1586                                         acc_mask ? (const uint8_t *)acc_mask
1587                                                  : (const uint8_t *)&nic_mask,
1588                                         sizeof(struct rte_flow_item_ipv4),
1589                                         error);
1590         if (ret < 0)
1591                 return ret;
1592         return 0;
1593 }
1594
1595 /**
1596  * Validate IPV6 item.
1597  *
1598  * @param[in] item
1599  *   Item specification.
1600  * @param[in] item_flags
1601  *   Bit-fields that holds the items detected until now.
1602  * @param[in] acc_mask
1603  *   Acceptable mask, if NULL default internal default mask
1604  *   will be used to check whether item fields are supported.
1605  * @param[out] error
1606  *   Pointer to error structure.
1607  *
1608  * @return
1609  *   0 on success, a negative errno value otherwise and rte_errno is set.
1610  */
1611 int
1612 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1613                              uint64_t item_flags,
1614                              uint64_t last_item,
1615                              uint16_t ether_type,
1616                              const struct rte_flow_item_ipv6 *acc_mask,
1617                              struct rte_flow_error *error)
1618 {
1619         const struct rte_flow_item_ipv6 *mask = item->mask;
1620         const struct rte_flow_item_ipv6 *spec = item->spec;
1621         const struct rte_flow_item_ipv6 nic_mask = {
1622                 .hdr = {
1623                         .src_addr =
1624                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1625                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1626                         .dst_addr =
1627                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1628                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1629                         .vtc_flow = RTE_BE32(0xffffffff),
1630                         .proto = 0xff,
1631                         .hop_limits = 0xff,
1632                 },
1633         };
1634         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1635         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1636                                       MLX5_FLOW_LAYER_OUTER_L3;
1637         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1638                                       MLX5_FLOW_LAYER_OUTER_L4;
1639         int ret;
1640         uint8_t next_proto = 0xFF;
1641         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1642                                   MLX5_FLOW_LAYER_OUTER_VLAN |
1643                                   MLX5_FLOW_LAYER_INNER_VLAN);
1644
1645         if ((last_item & l2_vlan) && ether_type &&
1646             ether_type != RTE_ETHER_TYPE_IPV6)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1649                                           "IPv6 cannot follow L2/VLAN layer "
1650                                           "which ether type is not IPv6");
1651         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
1652                 if (mask && spec)
1653                         next_proto = mask->hdr.proto & spec->hdr.proto;
1654                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1655                         return rte_flow_error_set(error, EINVAL,
1656                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1657                                                   item,
1658                                                   "multiple tunnel "
1659                                                   "not supported");
1660         }
1661         if (item_flags & MLX5_FLOW_LAYER_IPIP)
1662                 return rte_flow_error_set(error, EINVAL,
1663                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1664                                           "wrong tunnel type - IPv4 specified "
1665                                           "but IPv6 item provided");
1666         if (item_flags & l3m)
1667                 return rte_flow_error_set(error, ENOTSUP,
1668                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1669                                           "multiple L3 layers not supported");
1670         else if (item_flags & l4m)
1671                 return rte_flow_error_set(error, EINVAL,
1672                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1673                                           "L3 cannot follow an L4 layer.");
1674         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1675                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1676                 return rte_flow_error_set(error, EINVAL,
1677                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1678                                           "L3 cannot follow an NVGRE layer.");
1679         if (!mask)
1680                 mask = &rte_flow_item_ipv6_mask;
1681         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1682                                         acc_mask ? (const uint8_t *)acc_mask
1683                                                  : (const uint8_t *)&nic_mask,
1684                                         sizeof(struct rte_flow_item_ipv6),
1685                                         error);
1686         if (ret < 0)
1687                 return ret;
1688         return 0;
1689 }
1690
1691 /**
1692  * Validate UDP item.
1693  *
1694  * @param[in] item
1695  *   Item specification.
1696  * @param[in] item_flags
1697  *   Bit-fields that holds the items detected until now.
1698  * @param[in] target_protocol
1699  *   The next protocol in the previous item.
1700  * @param[in] flow_mask
1701  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
1702  * @param[out] error
1703  *   Pointer to error structure.
1704  *
1705  * @return
1706  *   0 on success, a negative errno value otherwise and rte_errno is set.
1707  */
1708 int
1709 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1710                             uint64_t item_flags,
1711                             uint8_t target_protocol,
1712                             struct rte_flow_error *error)
1713 {
1714         const struct rte_flow_item_udp *mask = item->mask;
1715         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1716         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1717                                       MLX5_FLOW_LAYER_OUTER_L3;
1718         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1719                                       MLX5_FLOW_LAYER_OUTER_L4;
1720         int ret;
1721
1722         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1723                 return rte_flow_error_set(error, EINVAL,
1724                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1725                                           "protocol filtering not compatible"
1726                                           " with UDP layer");
1727         if (!(item_flags & l3m))
1728                 return rte_flow_error_set(error, EINVAL,
1729                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1730                                           "L3 is mandatory to filter on L4");
1731         if (item_flags & l4m)
1732                 return rte_flow_error_set(error, EINVAL,
1733                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1734                                           "multiple L4 layers not supported");
1735         if (!mask)
1736                 mask = &rte_flow_item_udp_mask;
1737         ret = mlx5_flow_item_acceptable
1738                 (item, (const uint8_t *)mask,
1739                  (const uint8_t *)&rte_flow_item_udp_mask,
1740                  sizeof(struct rte_flow_item_udp), error);
1741         if (ret < 0)
1742                 return ret;
1743         return 0;
1744 }
1745
1746 /**
1747  * Validate TCP item.
1748  *
1749  * @param[in] item
1750  *   Item specification.
1751  * @param[in] item_flags
1752  *   Bit-fields that holds the items detected until now.
1753  * @param[in] target_protocol
1754  *   The next protocol in the previous item.
1755  * @param[out] error
1756  *   Pointer to error structure.
1757  *
1758  * @return
1759  *   0 on success, a negative errno value otherwise and rte_errno is set.
1760  */
1761 int
1762 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1763                             uint64_t item_flags,
1764                             uint8_t target_protocol,
1765                             const struct rte_flow_item_tcp *flow_mask,
1766                             struct rte_flow_error *error)
1767 {
1768         const struct rte_flow_item_tcp *mask = item->mask;
1769         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1770         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1771                                       MLX5_FLOW_LAYER_OUTER_L3;
1772         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1773                                       MLX5_FLOW_LAYER_OUTER_L4;
1774         int ret;
1775
1776         assert(flow_mask);
1777         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1778                 return rte_flow_error_set(error, EINVAL,
1779                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1780                                           "protocol filtering not compatible"
1781                                           " with TCP layer");
1782         if (!(item_flags & l3m))
1783                 return rte_flow_error_set(error, EINVAL,
1784                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1785                                           "L3 is mandatory to filter on L4");
1786         if (item_flags & l4m)
1787                 return rte_flow_error_set(error, EINVAL,
1788                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1789                                           "multiple L4 layers not supported");
1790         if (!mask)
1791                 mask = &rte_flow_item_tcp_mask;
1792         ret = mlx5_flow_item_acceptable
1793                 (item, (const uint8_t *)mask,
1794                  (const uint8_t *)flow_mask,
1795                  sizeof(struct rte_flow_item_tcp), error);
1796         if (ret < 0)
1797                 return ret;
1798         return 0;
1799 }
1800
1801 /**
1802  * Validate VXLAN item.
1803  *
1804  * @param[in] item
1805  *   Item specification.
1806  * @param[in] item_flags
1807  *   Bit-fields that holds the items detected until now.
1808  * @param[in] target_protocol
1809  *   The next protocol in the previous item.
1810  * @param[out] error
1811  *   Pointer to error structure.
1812  *
1813  * @return
1814  *   0 on success, a negative errno value otherwise and rte_errno is set.
1815  */
1816 int
1817 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1818                               uint64_t item_flags,
1819                               struct rte_flow_error *error)
1820 {
1821         const struct rte_flow_item_vxlan *spec = item->spec;
1822         const struct rte_flow_item_vxlan *mask = item->mask;
1823         int ret;
1824         union vni {
1825                 uint32_t vlan_id;
1826                 uint8_t vni[4];
1827         } id = { .vlan_id = 0, };
1828         uint32_t vlan_id = 0;
1829
1830
1831         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1832                 return rte_flow_error_set(error, ENOTSUP,
1833                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1834                                           "multiple tunnel layers not"
1835                                           " supported");
1836         /*
1837          * Verify only UDPv4 is present as defined in
1838          * https://tools.ietf.org/html/rfc7348
1839          */
1840         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1841                 return rte_flow_error_set(error, EINVAL,
1842                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1843                                           "no outer UDP layer found");
1844         if (!mask)
1845                 mask = &rte_flow_item_vxlan_mask;
1846         ret = mlx5_flow_item_acceptable
1847                 (item, (const uint8_t *)mask,
1848                  (const uint8_t *)&rte_flow_item_vxlan_mask,
1849                  sizeof(struct rte_flow_item_vxlan),
1850                  error);
1851         if (ret < 0)
1852                 return ret;
1853         if (spec) {
1854                 memcpy(&id.vni[1], spec->vni, 3);
1855                 vlan_id = id.vlan_id;
1856                 memcpy(&id.vni[1], mask->vni, 3);
1857                 vlan_id &= id.vlan_id;
1858         }
1859         /*
1860          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
1861          * only this layer is defined in the Verbs specification it is
1862          * interpreted as wildcard and all packets will match this
1863          * rule, if it follows a full stack layer (ex: eth / ipv4 /
1864          * udp), all packets matching the layers before will also
1865          * match this rule.  To avoid such situation, VNI 0 is
1866          * currently refused.
1867          */
1868         if (!vlan_id)
1869                 return rte_flow_error_set(error, ENOTSUP,
1870                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1871                                           "VXLAN vni cannot be 0");
1872         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1873                 return rte_flow_error_set(error, ENOTSUP,
1874                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1875                                           "VXLAN tunnel must be fully defined");
1876         return 0;
1877 }
1878
1879 /**
1880  * Validate VXLAN_GPE item.
1881  *
1882  * @param[in] item
1883  *   Item specification.
1884  * @param[in] item_flags
1885  *   Bit-fields that holds the items detected until now.
1886  * @param[in] priv
1887  *   Pointer to the private data structure.
1888  * @param[in] target_protocol
1889  *   The next protocol in the previous item.
1890  * @param[out] error
1891  *   Pointer to error structure.
1892  *
1893  * @return
1894  *   0 on success, a negative errno value otherwise and rte_errno is set.
1895  */
1896 int
1897 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1898                                   uint64_t item_flags,
1899                                   struct rte_eth_dev *dev,
1900                                   struct rte_flow_error *error)
1901 {
1902         struct mlx5_priv *priv = dev->data->dev_private;
1903         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1904         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1905         int ret;
1906         union vni {
1907                 uint32_t vlan_id;
1908                 uint8_t vni[4];
1909         } id = { .vlan_id = 0, };
1910         uint32_t vlan_id = 0;
1911
1912         if (!priv->config.l3_vxlan_en)
1913                 return rte_flow_error_set(error, ENOTSUP,
1914                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1915                                           "L3 VXLAN is not enabled by device"
1916                                           " parameter and/or not configured in"
1917                                           " firmware");
1918         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1919                 return rte_flow_error_set(error, ENOTSUP,
1920                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1921                                           "multiple tunnel layers not"
1922                                           " supported");
1923         /*
1924          * Verify only UDPv4 is present as defined in
1925          * https://tools.ietf.org/html/rfc7348
1926          */
1927         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1928                 return rte_flow_error_set(error, EINVAL,
1929                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1930                                           "no outer UDP layer found");
1931         if (!mask)
1932                 mask = &rte_flow_item_vxlan_gpe_mask;
1933         ret = mlx5_flow_item_acceptable
1934                 (item, (const uint8_t *)mask,
1935                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1936                  sizeof(struct rte_flow_item_vxlan_gpe),
1937                  error);
1938         if (ret < 0)
1939                 return ret;
1940         if (spec) {
1941                 if (spec->protocol)
1942                         return rte_flow_error_set(error, ENOTSUP,
1943                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1944                                                   item,
1945                                                   "VxLAN-GPE protocol"
1946                                                   " not supported");
1947                 memcpy(&id.vni[1], spec->vni, 3);
1948                 vlan_id = id.vlan_id;
1949                 memcpy(&id.vni[1], mask->vni, 3);
1950                 vlan_id &= id.vlan_id;
1951         }
1952         /*
1953          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1954          * layer is defined in the Verbs specification it is interpreted as
1955          * wildcard and all packets will match this rule, if it follows a full
1956          * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1957          * before will also match this rule.  To avoid such situation, VNI 0
1958          * is currently refused.
1959          */
1960         if (!vlan_id)
1961                 return rte_flow_error_set(error, ENOTSUP,
1962                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1963                                           "VXLAN-GPE vni cannot be 0");
1964         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1965                 return rte_flow_error_set(error, ENOTSUP,
1966                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1967                                           "VXLAN-GPE tunnel must be fully"
1968                                           " defined");
1969         return 0;
1970 }
1971 /**
1972  * Validate GRE Key item.
1973  *
1974  * @param[in] item
1975  *   Item specification.
1976  * @param[in] item_flags
1977  *   Bit flags to mark detected items.
1978  * @param[in] gre_item
1979  *   Pointer to gre_item
1980  * @param[out] error
1981  *   Pointer to error structure.
1982  *
1983  * @return
1984  *   0 on success, a negative errno value otherwise and rte_errno is set.
1985  */
1986 int
1987 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
1988                                 uint64_t item_flags,
1989                                 const struct rte_flow_item *gre_item,
1990                                 struct rte_flow_error *error)
1991 {
1992         const rte_be32_t *mask = item->mask;
1993         int ret = 0;
1994         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
1995         const struct rte_flow_item_gre *gre_spec;
1996         const struct rte_flow_item_gre *gre_mask;
1997
1998         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
1999                 return rte_flow_error_set(error, ENOTSUP,
2000                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2001                                           "Multiple GRE key not support");
2002         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2003                 return rte_flow_error_set(error, ENOTSUP,
2004                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2005                                           "No preceding GRE header");
2006         if (item_flags & MLX5_FLOW_LAYER_INNER)
2007                 return rte_flow_error_set(error, ENOTSUP,
2008                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2009                                           "GRE key following a wrong item");
2010         gre_mask = gre_item->mask;
2011         if (!gre_mask)
2012                 gre_mask = &rte_flow_item_gre_mask;
2013         gre_spec = gre_item->spec;
2014         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2015                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2016                 return rte_flow_error_set(error, EINVAL,
2017                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2018                                           "Key bit must be on");
2019
2020         if (!mask)
2021                 mask = &gre_key_default_mask;
2022         ret = mlx5_flow_item_acceptable
2023                 (item, (const uint8_t *)mask,
2024                  (const uint8_t *)&gre_key_default_mask,
2025                  sizeof(rte_be32_t), error);
2026         return ret;
2027 }
2028
2029 /**
2030  * Validate GRE item.
2031  *
2032  * @param[in] item
2033  *   Item specification.
2034  * @param[in] item_flags
2035  *   Bit flags to mark detected items.
2036  * @param[in] target_protocol
2037  *   The next protocol in the previous item.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 int
2045 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2046                             uint64_t item_flags,
2047                             uint8_t target_protocol,
2048                             struct rte_flow_error *error)
2049 {
2050         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2051         const struct rte_flow_item_gre *mask = item->mask;
2052         int ret;
2053         const struct rte_flow_item_gre nic_mask = {
2054                 .c_rsvd0_ver = RTE_BE16(0xB000),
2055                 .protocol = RTE_BE16(UINT16_MAX),
2056         };
2057
2058         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2059                 return rte_flow_error_set(error, EINVAL,
2060                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2061                                           "protocol filtering not compatible"
2062                                           " with this GRE layer");
2063         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2064                 return rte_flow_error_set(error, ENOTSUP,
2065                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2066                                           "multiple tunnel layers not"
2067                                           " supported");
2068         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2069                 return rte_flow_error_set(error, ENOTSUP,
2070                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2071                                           "L3 Layer is missing");
2072         if (!mask)
2073                 mask = &rte_flow_item_gre_mask;
2074         ret = mlx5_flow_item_acceptable
2075                 (item, (const uint8_t *)mask,
2076                  (const uint8_t *)&nic_mask,
2077                  sizeof(struct rte_flow_item_gre), error);
2078         if (ret < 0)
2079                 return ret;
2080 #ifndef HAVE_MLX5DV_DR
2081 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2082         if (spec && (spec->protocol & mask->protocol))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2085                                           "without MPLS support the"
2086                                           " specification cannot be used for"
2087                                           " filtering");
2088 #endif
2089 #endif
2090         return 0;
2091 }
2092
2093 /**
2094  * Validate Geneve item.
2095  *
2096  * @param[in] item
2097  *   Item specification.
2098  * @param[in] itemFlags
2099  *   Bit-fields that holds the items detected until now.
2100  * @param[in] enPriv
2101  *   Pointer to the private data structure.
2102  * @param[out] error
2103  *   Pointer to error structure.
2104  *
2105  * @return
2106  *   0 on success, a negative errno value otherwise and rte_errno is set.
2107  */
2108
2109 int
2110 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2111                                uint64_t item_flags,
2112                                struct rte_eth_dev *dev,
2113                                struct rte_flow_error *error)
2114 {
2115         struct mlx5_priv *priv = dev->data->dev_private;
2116         const struct rte_flow_item_geneve *spec = item->spec;
2117         const struct rte_flow_item_geneve *mask = item->mask;
2118         int ret;
2119         uint16_t gbhdr;
2120         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2121                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2122         const struct rte_flow_item_geneve nic_mask = {
2123                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2124                 .vni = "\xff\xff\xff",
2125                 .protocol = RTE_BE16(UINT16_MAX),
2126         };
2127
2128         if (!(priv->config.hca_attr.flex_parser_protocols &
2129               MLX5_HCA_FLEX_GENEVE_ENABLED) ||
2130             !priv->config.hca_attr.tunnel_stateless_geneve_rx)
2131                 return rte_flow_error_set(error, ENOTSUP,
2132                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2133                                           "L3 Geneve is not enabled by device"
2134                                           " parameter and/or not configured in"
2135                                           " firmware");
2136         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2137                 return rte_flow_error_set(error, ENOTSUP,
2138                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2139                                           "multiple tunnel layers not"
2140                                           " supported");
2141         /*
2142          * Verify only UDPv4 is present as defined in
2143          * https://tools.ietf.org/html/rfc7348
2144          */
2145         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2146                 return rte_flow_error_set(error, EINVAL,
2147                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2148                                           "no outer UDP layer found");
2149         if (!mask)
2150                 mask = &rte_flow_item_geneve_mask;
2151         ret = mlx5_flow_item_acceptable
2152                                   (item, (const uint8_t *)mask,
2153                                    (const uint8_t *)&nic_mask,
2154                                    sizeof(struct rte_flow_item_geneve), error);
2155         if (ret)
2156                 return ret;
2157         if (spec) {
2158                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2159                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2160                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2161                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2162                         return rte_flow_error_set(error, ENOTSUP,
2163                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2164                                                   item,
2165                                                   "Geneve protocol unsupported"
2166                                                   " fields are being used");
2167                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2168                         return rte_flow_error_set
2169                                         (error, ENOTSUP,
2170                                          RTE_FLOW_ERROR_TYPE_ITEM,
2171                                          item,
2172                                          "Unsupported Geneve options length");
2173         }
2174         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2175                 return rte_flow_error_set
2176                                     (error, ENOTSUP,
2177                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2178                                      "Geneve tunnel must be fully defined");
2179         return 0;
2180 }
2181
2182 /**
2183  * Validate MPLS item.
2184  *
2185  * @param[in] dev
2186  *   Pointer to the rte_eth_dev structure.
2187  * @param[in] item
2188  *   Item specification.
2189  * @param[in] item_flags
2190  *   Bit-fields that holds the items detected until now.
2191  * @param[in] prev_layer
2192  *   The protocol layer indicated in previous item.
2193  * @param[out] error
2194  *   Pointer to error structure.
2195  *
2196  * @return
2197  *   0 on success, a negative errno value otherwise and rte_errno is set.
2198  */
2199 int
2200 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2201                              const struct rte_flow_item *item __rte_unused,
2202                              uint64_t item_flags __rte_unused,
2203                              uint64_t prev_layer __rte_unused,
2204                              struct rte_flow_error *error)
2205 {
2206 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2207         const struct rte_flow_item_mpls *mask = item->mask;
2208         struct mlx5_priv *priv = dev->data->dev_private;
2209         int ret;
2210
2211         if (!priv->config.mpls_en)
2212                 return rte_flow_error_set(error, ENOTSUP,
2213                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2214                                           "MPLS not supported or"
2215                                           " disabled in firmware"
2216                                           " configuration.");
2217         /* MPLS over IP, UDP, GRE is allowed */
2218         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2219                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
2220                             MLX5_FLOW_LAYER_GRE)))
2221                 return rte_flow_error_set(error, EINVAL,
2222                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2223                                           "protocol filtering not compatible"
2224                                           " with MPLS layer");
2225         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2226         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2227             !(item_flags & MLX5_FLOW_LAYER_GRE))
2228                 return rte_flow_error_set(error, ENOTSUP,
2229                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2230                                           "multiple tunnel layers not"
2231                                           " supported");
2232         if (!mask)
2233                 mask = &rte_flow_item_mpls_mask;
2234         ret = mlx5_flow_item_acceptable
2235                 (item, (const uint8_t *)mask,
2236                  (const uint8_t *)&rte_flow_item_mpls_mask,
2237                  sizeof(struct rte_flow_item_mpls), error);
2238         if (ret < 0)
2239                 return ret;
2240         return 0;
2241 #endif
2242         return rte_flow_error_set(error, ENOTSUP,
2243                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
2244                                   "MPLS is not supported by Verbs, please"
2245                                   " update.");
2246 }
2247
2248 /**
2249  * Validate NVGRE item.
2250  *
2251  * @param[in] item
2252  *   Item specification.
2253  * @param[in] item_flags
2254  *   Bit flags to mark detected items.
2255  * @param[in] target_protocol
2256  *   The next protocol in the previous item.
2257  * @param[out] error
2258  *   Pointer to error structure.
2259  *
2260  * @return
2261  *   0 on success, a negative errno value otherwise and rte_errno is set.
2262  */
2263 int
2264 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2265                               uint64_t item_flags,
2266                               uint8_t target_protocol,
2267                               struct rte_flow_error *error)
2268 {
2269         const struct rte_flow_item_nvgre *mask = item->mask;
2270         int ret;
2271
2272         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2273                 return rte_flow_error_set(error, EINVAL,
2274                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2275                                           "protocol filtering not compatible"
2276                                           " with this GRE layer");
2277         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2278                 return rte_flow_error_set(error, ENOTSUP,
2279                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2280                                           "multiple tunnel layers not"
2281                                           " supported");
2282         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2283                 return rte_flow_error_set(error, ENOTSUP,
2284                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2285                                           "L3 Layer is missing");
2286         if (!mask)
2287                 mask = &rte_flow_item_nvgre_mask;
2288         ret = mlx5_flow_item_acceptable
2289                 (item, (const uint8_t *)mask,
2290                  (const uint8_t *)&rte_flow_item_nvgre_mask,
2291                  sizeof(struct rte_flow_item_nvgre), error);
2292         if (ret < 0)
2293                 return ret;
2294         return 0;
2295 }
2296
2297 /* Allocate unique ID for the split Q/RSS subflows. */
2298 static uint32_t
2299 flow_qrss_get_id(struct rte_eth_dev *dev)
2300 {
2301         struct mlx5_priv *priv = dev->data->dev_private;
2302         uint32_t qrss_id, ret;
2303
2304         ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2305         if (ret)
2306                 return 0;
2307         assert(qrss_id);
2308         return qrss_id;
2309 }
2310
2311 /* Free unique ID for the split Q/RSS subflows. */
2312 static void
2313 flow_qrss_free_id(struct rte_eth_dev *dev,  uint32_t qrss_id)
2314 {
2315         struct mlx5_priv *priv = dev->data->dev_private;
2316
2317         if (qrss_id)
2318                 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2319 }
2320
2321 /**
2322  * Release resource related QUEUE/RSS action split.
2323  *
2324  * @param dev
2325  *   Pointer to Ethernet device.
2326  * @param flow
2327  *   Flow to release id's from.
2328  */
2329 static void
2330 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2331                              struct rte_flow *flow)
2332 {
2333         struct mlx5_flow *dev_flow;
2334
2335         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
2336                 if (dev_flow->qrss_id)
2337                         flow_qrss_free_id(dev, dev_flow->qrss_id);
2338 }
2339
2340 static int
2341 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2342                    const struct rte_flow_attr *attr __rte_unused,
2343                    const struct rte_flow_item items[] __rte_unused,
2344                    const struct rte_flow_action actions[] __rte_unused,
2345                    bool external __rte_unused,
2346                    struct rte_flow_error *error)
2347 {
2348         return rte_flow_error_set(error, ENOTSUP,
2349                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2350 }
2351
2352 static struct mlx5_flow *
2353 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
2354                   const struct rte_flow_item items[] __rte_unused,
2355                   const struct rte_flow_action actions[] __rte_unused,
2356                   struct rte_flow_error *error)
2357 {
2358         rte_flow_error_set(error, ENOTSUP,
2359                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2360         return NULL;
2361 }
2362
2363 static int
2364 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2365                     struct mlx5_flow *dev_flow __rte_unused,
2366                     const struct rte_flow_attr *attr __rte_unused,
2367                     const struct rte_flow_item items[] __rte_unused,
2368                     const struct rte_flow_action actions[] __rte_unused,
2369                     struct rte_flow_error *error)
2370 {
2371         return rte_flow_error_set(error, ENOTSUP,
2372                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2373 }
2374
2375 static int
2376 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2377                 struct rte_flow *flow __rte_unused,
2378                 struct rte_flow_error *error)
2379 {
2380         return rte_flow_error_set(error, ENOTSUP,
2381                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2382 }
2383
2384 static void
2385 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2386                  struct rte_flow *flow __rte_unused)
2387 {
2388 }
2389
2390 static void
2391 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2392                   struct rte_flow *flow __rte_unused)
2393 {
2394 }
2395
2396 static int
2397 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2398                 struct rte_flow *flow __rte_unused,
2399                 const struct rte_flow_action *actions __rte_unused,
2400                 void *data __rte_unused,
2401                 struct rte_flow_error *error)
2402 {
2403         return rte_flow_error_set(error, ENOTSUP,
2404                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2405 }
2406
2407 /* Void driver to protect from null pointer reference. */
2408 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2409         .validate = flow_null_validate,
2410         .prepare = flow_null_prepare,
2411         .translate = flow_null_translate,
2412         .apply = flow_null_apply,
2413         .remove = flow_null_remove,
2414         .destroy = flow_null_destroy,
2415         .query = flow_null_query,
2416 };
2417
2418 /**
2419  * Select flow driver type according to flow attributes and device
2420  * configuration.
2421  *
2422  * @param[in] dev
2423  *   Pointer to the dev structure.
2424  * @param[in] attr
2425  *   Pointer to the flow attributes.
2426  *
2427  * @return
2428  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2429  */
2430 static enum mlx5_flow_drv_type
2431 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2432 {
2433         struct mlx5_priv *priv = dev->data->dev_private;
2434         enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
2435
2436         if (attr->transfer && priv->config.dv_esw_en)
2437                 type = MLX5_FLOW_TYPE_DV;
2438         if (!attr->transfer)
2439                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2440                                                  MLX5_FLOW_TYPE_VERBS;
2441         return type;
2442 }
2443
2444 #define flow_get_drv_ops(type) flow_drv_ops[type]
2445
2446 /**
2447  * Flow driver validation API. This abstracts calling driver specific functions.
2448  * The type of flow driver is determined according to flow attributes.
2449  *
2450  * @param[in] dev
2451  *   Pointer to the dev structure.
2452  * @param[in] attr
2453  *   Pointer to the flow attributes.
2454  * @param[in] items
2455  *   Pointer to the list of items.
2456  * @param[in] actions
2457  *   Pointer to the list of actions.
2458  * @param[in] external
2459  *   This flow rule is created by request external to PMD.
2460  * @param[out] error
2461  *   Pointer to the error structure.
2462  *
2463  * @return
2464  *   0 on success, a negative errno value otherwise and rte_errno is set.
2465  */
2466 static inline int
2467 flow_drv_validate(struct rte_eth_dev *dev,
2468                   const struct rte_flow_attr *attr,
2469                   const struct rte_flow_item items[],
2470                   const struct rte_flow_action actions[],
2471                   bool external, struct rte_flow_error *error)
2472 {
2473         const struct mlx5_flow_driver_ops *fops;
2474         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2475
2476         fops = flow_get_drv_ops(type);
2477         return fops->validate(dev, attr, items, actions, external, error);
2478 }
2479
2480 /**
2481  * Flow driver preparation API. This abstracts calling driver specific
2482  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2483  * calculates the size of memory required for device flow, allocates the memory,
2484  * initializes the device flow and returns the pointer.
2485  *
2486  * @note
2487  *   This function initializes device flow structure such as dv or verbs in
2488  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
2489  *   rest. For example, adding returning device flow to flow->dev_flow list and
2490  *   setting backward reference to the flow should be done out of this function.
2491  *   layers field is not filled either.
2492  *
2493  * @param[in] attr
2494  *   Pointer to the flow attributes.
2495  * @param[in] items
2496  *   Pointer to the list of items.
2497  * @param[in] actions
2498  *   Pointer to the list of actions.
2499  * @param[out] error
2500  *   Pointer to the error structure.
2501  *
2502  * @return
2503  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
2504  */
2505 static inline struct mlx5_flow *
2506 flow_drv_prepare(const struct rte_flow *flow,
2507                  const struct rte_flow_attr *attr,
2508                  const struct rte_flow_item items[],
2509                  const struct rte_flow_action actions[],
2510                  struct rte_flow_error *error)
2511 {
2512         const struct mlx5_flow_driver_ops *fops;
2513         enum mlx5_flow_drv_type type = flow->drv_type;
2514
2515         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2516         fops = flow_get_drv_ops(type);
2517         return fops->prepare(attr, items, actions, error);
2518 }
2519
2520 /**
2521  * Flow driver translation API. This abstracts calling driver specific
2522  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2523  * translates a generic flow into a driver flow. flow_drv_prepare() must
2524  * precede.
2525  *
2526  * @note
2527  *   dev_flow->layers could be filled as a result of parsing during translation
2528  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
2529  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
2530  *   flow->actions could be overwritten even though all the expanded dev_flows
2531  *   have the same actions.
2532  *
2533  * @param[in] dev
2534  *   Pointer to the rte dev structure.
2535  * @param[in, out] dev_flow
2536  *   Pointer to the mlx5 flow.
2537  * @param[in] attr
2538  *   Pointer to the flow attributes.
2539  * @param[in] items
2540  *   Pointer to the list of items.
2541  * @param[in] actions
2542  *   Pointer to the list of actions.
2543  * @param[out] error
2544  *   Pointer to the error structure.
2545  *
2546  * @return
2547  *   0 on success, a negative errno value otherwise and rte_errno is set.
2548  */
2549 static inline int
2550 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2551                    const struct rte_flow_attr *attr,
2552                    const struct rte_flow_item items[],
2553                    const struct rte_flow_action actions[],
2554                    struct rte_flow_error *error)
2555 {
2556         const struct mlx5_flow_driver_ops *fops;
2557         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
2558
2559         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2560         fops = flow_get_drv_ops(type);
2561         return fops->translate(dev, dev_flow, attr, items, actions, error);
2562 }
2563
2564 /**
2565  * Flow driver apply API. This abstracts calling driver specific functions.
2566  * Parent flow (rte_flow) should have driver type (drv_type). It applies
2567  * translated driver flows on to device. flow_drv_translate() must precede.
2568  *
2569  * @param[in] dev
2570  *   Pointer to Ethernet device structure.
2571  * @param[in, out] flow
2572  *   Pointer to flow structure.
2573  * @param[out] error
2574  *   Pointer to error structure.
2575  *
2576  * @return
2577  *   0 on success, a negative errno value otherwise and rte_errno is set.
2578  */
2579 static inline int
2580 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
2581                struct rte_flow_error *error)
2582 {
2583         const struct mlx5_flow_driver_ops *fops;
2584         enum mlx5_flow_drv_type type = flow->drv_type;
2585
2586         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2587         fops = flow_get_drv_ops(type);
2588         return fops->apply(dev, flow, error);
2589 }
2590
2591 /**
2592  * Flow driver remove API. This abstracts calling driver specific functions.
2593  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2594  * on device. All the resources of the flow should be freed by calling
2595  * flow_drv_destroy().
2596  *
2597  * @param[in] dev
2598  *   Pointer to Ethernet device.
2599  * @param[in, out] flow
2600  *   Pointer to flow structure.
2601  */
2602 static inline void
2603 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
2604 {
2605         const struct mlx5_flow_driver_ops *fops;
2606         enum mlx5_flow_drv_type type = flow->drv_type;
2607
2608         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2609         fops = flow_get_drv_ops(type);
2610         fops->remove(dev, flow);
2611 }
2612
2613 /**
2614  * Flow driver destroy API. This abstracts calling driver specific functions.
2615  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2616  * on device and releases resources of the flow.
2617  *
2618  * @param[in] dev
2619  *   Pointer to Ethernet device.
2620  * @param[in, out] flow
2621  *   Pointer to flow structure.
2622  */
2623 static inline void
2624 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
2625 {
2626         const struct mlx5_flow_driver_ops *fops;
2627         enum mlx5_flow_drv_type type = flow->drv_type;
2628
2629         flow_mreg_split_qrss_release(dev, flow);
2630         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2631         fops = flow_get_drv_ops(type);
2632         fops->destroy(dev, flow);
2633 }
2634
2635 /**
2636  * Validate a flow supported by the NIC.
2637  *
2638  * @see rte_flow_validate()
2639  * @see rte_flow_ops
2640  */
2641 int
2642 mlx5_flow_validate(struct rte_eth_dev *dev,
2643                    const struct rte_flow_attr *attr,
2644                    const struct rte_flow_item items[],
2645                    const struct rte_flow_action actions[],
2646                    struct rte_flow_error *error)
2647 {
2648         int ret;
2649
2650         ret = flow_drv_validate(dev, attr, items, actions, true, error);
2651         if (ret < 0)
2652                 return ret;
2653         return 0;
2654 }
2655
2656 /**
2657  * Get port id item from the item list.
2658  *
2659  * @param[in] item
2660  *   Pointer to the list of items.
2661  *
2662  * @return
2663  *   Pointer to the port id item if exist, else return NULL.
2664  */
2665 static const struct rte_flow_item *
2666 find_port_id_item(const struct rte_flow_item *item)
2667 {
2668         assert(item);
2669         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2670                 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID)
2671                         return item;
2672         }
2673         return NULL;
2674 }
2675
2676 /**
2677  * Get RSS action from the action list.
2678  *
2679  * @param[in] actions
2680  *   Pointer to the list of actions.
2681  *
2682  * @return
2683  *   Pointer to the RSS action if exist, else return NULL.
2684  */
2685 static const struct rte_flow_action_rss*
2686 flow_get_rss_action(const struct rte_flow_action actions[])
2687 {
2688         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2689                 switch (actions->type) {
2690                 case RTE_FLOW_ACTION_TYPE_RSS:
2691                         return (const struct rte_flow_action_rss *)
2692                                actions->conf;
2693                 default:
2694                         break;
2695                 }
2696         }
2697         return NULL;
2698 }
2699
2700 static unsigned int
2701 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
2702 {
2703         const struct rte_flow_item *item;
2704         unsigned int has_vlan = 0;
2705
2706         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2707                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
2708                         has_vlan = 1;
2709                         break;
2710                 }
2711         }
2712         if (has_vlan)
2713                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
2714                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
2715         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
2716                                MLX5_EXPANSION_ROOT_OUTER;
2717 }
2718
2719 /**
2720  * Get QUEUE/RSS action from the action list.
2721  *
2722  * @param[in] actions
2723  *   Pointer to the list of actions.
2724  * @param[out] qrss
2725  *   Pointer to the return pointer.
2726  * @param[out] qrss_type
2727  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
2728  *   if no QUEUE/RSS is found.
2729  *
2730  * @return
2731  *   Total number of actions.
2732  */
2733 static int
2734 flow_parse_qrss_action(const struct rte_flow_action actions[],
2735                        const struct rte_flow_action **qrss)
2736 {
2737         int actions_n = 0;
2738
2739         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2740                 switch (actions->type) {
2741                 case RTE_FLOW_ACTION_TYPE_QUEUE:
2742                 case RTE_FLOW_ACTION_TYPE_RSS:
2743                         *qrss = actions;
2744                         break;
2745                 default:
2746                         break;
2747                 }
2748                 actions_n++;
2749         }
2750         /* Count RTE_FLOW_ACTION_TYPE_END. */
2751         return actions_n + 1;
2752 }
2753
2754 /**
2755  * Check meter action from the action list.
2756  *
2757  * @param[in] actions
2758  *   Pointer to the list of actions.
2759  * @param[out] mtr
2760  *   Pointer to the meter exist flag.
2761  *
2762  * @return
2763  *   Total number of actions.
2764  */
2765 static int
2766 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
2767 {
2768         int actions_n = 0;
2769
2770         assert(mtr);
2771         *mtr = 0;
2772         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2773                 switch (actions->type) {
2774                 case RTE_FLOW_ACTION_TYPE_METER:
2775                         *mtr = 1;
2776                         break;
2777                 default:
2778                         break;
2779                 }
2780                 actions_n++;
2781         }
2782         /* Count RTE_FLOW_ACTION_TYPE_END. */
2783         return actions_n + 1;
2784 }
2785
2786 /**
2787  * Check if the flow should be splited due to hairpin.
2788  * The reason for the split is that in current HW we can't
2789  * support encap on Rx, so if a flow have encap we move it
2790  * to Tx.
2791  *
2792  * @param dev
2793  *   Pointer to Ethernet device.
2794  * @param[in] attr
2795  *   Flow rule attributes.
2796  * @param[in] actions
2797  *   Associated actions (list terminated by the END action).
2798  *
2799  * @return
2800  *   > 0 the number of actions and the flow should be split,
2801  *   0 when no split required.
2802  */
2803 static int
2804 flow_check_hairpin_split(struct rte_eth_dev *dev,
2805                          const struct rte_flow_attr *attr,
2806                          const struct rte_flow_action actions[])
2807 {
2808         int queue_action = 0;
2809         int action_n = 0;
2810         int encap = 0;
2811         const struct rte_flow_action_queue *queue;
2812         const struct rte_flow_action_rss *rss;
2813         const struct rte_flow_action_raw_encap *raw_encap;
2814
2815         if (!attr->ingress)
2816                 return 0;
2817         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2818                 switch (actions->type) {
2819                 case RTE_FLOW_ACTION_TYPE_QUEUE:
2820                         queue = actions->conf;
2821                         if (queue == NULL)
2822                                 return 0;
2823                         if (mlx5_rxq_get_type(dev, queue->index) !=
2824                             MLX5_RXQ_TYPE_HAIRPIN)
2825                                 return 0;
2826                         queue_action = 1;
2827                         action_n++;
2828                         break;
2829                 case RTE_FLOW_ACTION_TYPE_RSS:
2830                         rss = actions->conf;
2831                         if (rss == NULL || rss->queue_num == 0)
2832                                 return 0;
2833                         if (mlx5_rxq_get_type(dev, rss->queue[0]) !=
2834                             MLX5_RXQ_TYPE_HAIRPIN)
2835                                 return 0;
2836                         queue_action = 1;
2837                         action_n++;
2838                         break;
2839                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2840                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2841                         encap = 1;
2842                         action_n++;
2843                         break;
2844                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2845                         raw_encap = actions->conf;
2846                         if (raw_encap->size >
2847                             (sizeof(struct rte_flow_item_eth) +
2848                              sizeof(struct rte_flow_item_ipv4)))
2849                                 encap = 1;
2850                         action_n++;
2851                         break;
2852                 default:
2853                         action_n++;
2854                         break;
2855                 }
2856         }
2857         if (encap == 1 && queue_action)
2858                 return action_n;
2859         return 0;
2860 }
2861
2862 /* Declare flow create/destroy prototype in advance. */
2863 static struct rte_flow *
2864 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
2865                  const struct rte_flow_attr *attr,
2866                  const struct rte_flow_item items[],
2867                  const struct rte_flow_action actions[],
2868                  bool external, struct rte_flow_error *error);
2869
2870 static void
2871 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
2872                   struct rte_flow *flow);
2873
2874 /**
2875  * Add a flow of copying flow metadata registers in RX_CP_TBL.
2876  *
2877  * As mark_id is unique, if there's already a registered flow for the mark_id,
2878  * return by increasing the reference counter of the resource. Otherwise, create
2879  * the resource (mcp_res) and flow.
2880  *
2881  * Flow looks like,
2882  *   - If ingress port is ANY and reg_c[1] is mark_id,
2883  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
2884  *
2885  * For default flow (zero mark_id), flow is like,
2886  *   - If ingress port is ANY,
2887  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
2888  *
2889  * @param dev
2890  *   Pointer to Ethernet device.
2891  * @param mark_id
2892  *   ID of MARK action, zero means default flow for META.
2893  * @param[out] error
2894  *   Perform verbose error reporting if not NULL.
2895  *
2896  * @return
2897  *   Associated resource on success, NULL otherwise and rte_errno is set.
2898  */
2899 static struct mlx5_flow_mreg_copy_resource *
2900 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
2901                           struct rte_flow_error *error)
2902 {
2903         struct mlx5_priv *priv = dev->data->dev_private;
2904         struct rte_flow_attr attr = {
2905                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
2906                 .ingress = 1,
2907         };
2908         struct mlx5_rte_flow_item_tag tag_spec = {
2909                 .data = mark_id,
2910         };
2911         struct rte_flow_item items[] = {
2912                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
2913         };
2914         struct rte_flow_action_mark ftag = {
2915                 .id = mark_id,
2916         };
2917         struct mlx5_flow_action_copy_mreg cp_mreg = {
2918                 .dst = REG_B,
2919                 .src = 0,
2920         };
2921         struct rte_flow_action_jump jump = {
2922                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
2923         };
2924         struct rte_flow_action actions[] = {
2925                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
2926         };
2927         struct mlx5_flow_mreg_copy_resource *mcp_res;
2928         int ret;
2929
2930         /* Fill the register fileds in the flow. */
2931         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2932         if (ret < 0)
2933                 return NULL;
2934         tag_spec.id = ret;
2935         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
2936         if (ret < 0)
2937                 return NULL;
2938         cp_mreg.src = ret;
2939         /* Check if already registered. */
2940         assert(priv->mreg_cp_tbl);
2941         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
2942         if (mcp_res) {
2943                 /* For non-default rule. */
2944                 if (mark_id != MLX5_DEFAULT_COPY_ID)
2945                         mcp_res->refcnt++;
2946                 assert(mark_id != MLX5_DEFAULT_COPY_ID || mcp_res->refcnt == 1);
2947                 return mcp_res;
2948         }
2949         /* Provide the full width of FLAG specific value. */
2950         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
2951                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
2952         /* Build a new flow. */
2953         if (mark_id != MLX5_DEFAULT_COPY_ID) {
2954                 items[0] = (struct rte_flow_item){
2955                         .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG,
2956                         .spec = &tag_spec,
2957                 };
2958                 items[1] = (struct rte_flow_item){
2959                         .type = RTE_FLOW_ITEM_TYPE_END,
2960                 };
2961                 actions[0] = (struct rte_flow_action){
2962                         .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK,
2963                         .conf = &ftag,
2964                 };
2965                 actions[1] = (struct rte_flow_action){
2966                         .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
2967                         .conf = &cp_mreg,
2968                 };
2969                 actions[2] = (struct rte_flow_action){
2970                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
2971                         .conf = &jump,
2972                 };
2973                 actions[3] = (struct rte_flow_action){
2974                         .type = RTE_FLOW_ACTION_TYPE_END,
2975                 };
2976         } else {
2977                 /* Default rule, wildcard match. */
2978                 attr.priority = MLX5_FLOW_PRIO_RSVD;
2979                 items[0] = (struct rte_flow_item){
2980                         .type = RTE_FLOW_ITEM_TYPE_END,
2981                 };
2982                 actions[0] = (struct rte_flow_action){
2983                         .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
2984                         .conf = &cp_mreg,
2985                 };
2986                 actions[1] = (struct rte_flow_action){
2987                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
2988                         .conf = &jump,
2989                 };
2990                 actions[2] = (struct rte_flow_action){
2991                         .type = RTE_FLOW_ACTION_TYPE_END,
2992                 };
2993         }
2994         /* Build a new entry. */
2995         mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0);
2996         if (!mcp_res) {
2997                 rte_errno = ENOMEM;
2998                 return NULL;
2999         }
3000         /*
3001          * The copy Flows are not included in any list. There
3002          * ones are referenced from other Flows and can not
3003          * be applied, removed, deleted in ardbitrary order
3004          * by list traversing.
3005          */
3006         mcp_res->flow = flow_list_create(dev, NULL, &attr, items,
3007                                          actions, false, error);
3008         if (!mcp_res->flow)
3009                 goto error;
3010         mcp_res->refcnt++;
3011         mcp_res->hlist_ent.key = mark_id;
3012         ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3013                                 &mcp_res->hlist_ent);
3014         assert(!ret);
3015         if (ret)
3016                 goto error;
3017         return mcp_res;
3018 error:
3019         if (mcp_res->flow)
3020                 flow_list_destroy(dev, NULL, mcp_res->flow);
3021         rte_free(mcp_res);
3022         return NULL;
3023 }
3024
3025 /**
3026  * Release flow in RX_CP_TBL.
3027  *
3028  * @param dev
3029  *   Pointer to Ethernet device.
3030  * @flow
3031  *   Parent flow for wich copying is provided.
3032  */
3033 static void
3034 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3035                           struct rte_flow *flow)
3036 {
3037         struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
3038         struct mlx5_priv *priv = dev->data->dev_private;
3039
3040         if (!mcp_res || !priv->mreg_cp_tbl)
3041                 return;
3042         if (flow->copy_applied) {
3043                 assert(mcp_res->appcnt);
3044                 flow->copy_applied = 0;
3045                 --mcp_res->appcnt;
3046                 if (!mcp_res->appcnt)
3047                         flow_drv_remove(dev, mcp_res->flow);
3048         }
3049         /*
3050          * We do not check availability of metadata registers here,
3051          * because copy resources are not allocated in this case.
3052          */
3053         if (--mcp_res->refcnt)
3054                 return;
3055         assert(mcp_res->flow);
3056         flow_list_destroy(dev, NULL, mcp_res->flow);
3057         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3058         rte_free(mcp_res);
3059         flow->mreg_copy = NULL;
3060 }
3061
3062 /**
3063  * Start flow in RX_CP_TBL.
3064  *
3065  * @param dev
3066  *   Pointer to Ethernet device.
3067  * @flow
3068  *   Parent flow for wich copying is provided.
3069  *
3070  * @return
3071  *   0 on success, a negative errno value otherwise and rte_errno is set.
3072  */
3073 static int
3074 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3075                             struct rte_flow *flow)
3076 {
3077         struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
3078         int ret;
3079
3080         if (!mcp_res || flow->copy_applied)
3081                 return 0;
3082         if (!mcp_res->appcnt) {
3083                 ret = flow_drv_apply(dev, mcp_res->flow, NULL);
3084                 if (ret)
3085                         return ret;
3086         }
3087         ++mcp_res->appcnt;
3088         flow->copy_applied = 1;
3089         return 0;
3090 }
3091
3092 /**
3093  * Stop flow in RX_CP_TBL.
3094  *
3095  * @param dev
3096  *   Pointer to Ethernet device.
3097  * @flow
3098  *   Parent flow for wich copying is provided.
3099  */
3100 static void
3101 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3102                            struct rte_flow *flow)
3103 {
3104         struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
3105
3106         if (!mcp_res || !flow->copy_applied)
3107                 return;
3108         assert(mcp_res->appcnt);
3109         --mcp_res->appcnt;
3110         flow->copy_applied = 0;
3111         if (!mcp_res->appcnt)
3112                 flow_drv_remove(dev, mcp_res->flow);
3113 }
3114
3115 /**
3116  * Remove the default copy action from RX_CP_TBL.
3117  *
3118  * @param dev
3119  *   Pointer to Ethernet device.
3120  */
3121 static void
3122 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3123 {
3124         struct mlx5_flow_mreg_copy_resource *mcp_res;
3125         struct mlx5_priv *priv = dev->data->dev_private;
3126
3127         /* Check if default flow is registered. */
3128         if (!priv->mreg_cp_tbl)
3129                 return;
3130         mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3131                                             MLX5_DEFAULT_COPY_ID);
3132         if (!mcp_res)
3133                 return;
3134         assert(mcp_res->flow);
3135         flow_list_destroy(dev, NULL, mcp_res->flow);
3136         mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3137         rte_free(mcp_res);
3138 }
3139
3140 /**
3141  * Add the default copy action in in RX_CP_TBL.
3142  *
3143  * @param dev
3144  *   Pointer to Ethernet device.
3145  * @param[out] error
3146  *   Perform verbose error reporting if not NULL.
3147  *
3148  * @return
3149  *   0 for success, negative value otherwise and rte_errno is set.
3150  */
3151 static int
3152 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3153                                   struct rte_flow_error *error)
3154 {
3155         struct mlx5_priv *priv = dev->data->dev_private;
3156         struct mlx5_flow_mreg_copy_resource *mcp_res;
3157
3158         /* Check whether extensive metadata feature is engaged. */
3159         if (!priv->config.dv_flow_en ||
3160             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3161             !mlx5_flow_ext_mreg_supported(dev) ||
3162             !priv->sh->dv_regc0_mask)
3163                 return 0;
3164         mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3165         if (!mcp_res)
3166                 return -rte_errno;
3167         return 0;
3168 }
3169
3170 /**
3171  * Add a flow of copying flow metadata registers in RX_CP_TBL.
3172  *
3173  * All the flow having Q/RSS action should be split by
3174  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3175  * performs the following,
3176  *   - CQE->flow_tag := reg_c[1] (MARK)
3177  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3178  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3179  * but there should be a flow per each MARK ID set by MARK action.
3180  *
3181  * For the aforementioned reason, if there's a MARK action in flow's action
3182  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3183  * the MARK ID to CQE's flow_tag like,
3184  *   - If reg_c[1] is mark_id,
3185  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3186  *
3187  * For SET_META action which stores value in reg_c[0], as the destination is
3188  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3189  * MARK ID means the default flow. The default flow looks like,
3190  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3191  *
3192  * @param dev
3193  *   Pointer to Ethernet device.
3194  * @param flow
3195  *   Pointer to flow structure.
3196  * @param[in] actions
3197  *   Pointer to the list of actions.
3198  * @param[out] error
3199  *   Perform verbose error reporting if not NULL.
3200  *
3201  * @return
3202  *   0 on success, negative value otherwise and rte_errno is set.
3203  */
3204 static int
3205 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3206                             struct rte_flow *flow,
3207                             const struct rte_flow_action *actions,
3208                             struct rte_flow_error *error)
3209 {
3210         struct mlx5_priv *priv = dev->data->dev_private;
3211         struct mlx5_dev_config *config = &priv->config;
3212         struct mlx5_flow_mreg_copy_resource *mcp_res;
3213         const struct rte_flow_action_mark *mark;
3214
3215         /* Check whether extensive metadata feature is engaged. */
3216         if (!config->dv_flow_en ||
3217             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3218             !mlx5_flow_ext_mreg_supported(dev) ||
3219             !priv->sh->dv_regc0_mask)
3220                 return 0;
3221         /* Find MARK action. */
3222         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3223                 switch (actions->type) {
3224                 case RTE_FLOW_ACTION_TYPE_FLAG:
3225                         mcp_res = flow_mreg_add_copy_action
3226                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3227                         if (!mcp_res)
3228                                 return -rte_errno;
3229                         flow->mreg_copy = mcp_res;
3230                         if (dev->data->dev_started) {
3231                                 mcp_res->appcnt++;
3232                                 flow->copy_applied = 1;
3233                         }
3234                         return 0;
3235                 case RTE_FLOW_ACTION_TYPE_MARK:
3236                         mark = (const struct rte_flow_action_mark *)
3237                                 actions->conf;
3238                         mcp_res =
3239                                 flow_mreg_add_copy_action(dev, mark->id, error);
3240                         if (!mcp_res)
3241                                 return -rte_errno;
3242                         flow->mreg_copy = mcp_res;
3243                         if (dev->data->dev_started) {
3244                                 mcp_res->appcnt++;
3245                                 flow->copy_applied = 1;
3246                         }
3247                         return 0;
3248                 default:
3249                         break;
3250                 }
3251         }
3252         return 0;
3253 }
3254
3255 #define MLX5_MAX_SPLIT_ACTIONS 24
3256 #define MLX5_MAX_SPLIT_ITEMS 24
3257
3258 /**
3259  * Split the hairpin flow.
3260  * Since HW can't support encap on Rx we move the encap to Tx.
3261  * If the count action is after the encap then we also
3262  * move the count action. in this case the count will also measure
3263  * the outer bytes.
3264  *
3265  * @param dev
3266  *   Pointer to Ethernet device.
3267  * @param[in] actions
3268  *   Associated actions (list terminated by the END action).
3269  * @param[out] actions_rx
3270  *   Rx flow actions.
3271  * @param[out] actions_tx
3272  *   Tx flow actions..
3273  * @param[out] pattern_tx
3274  *   The pattern items for the Tx flow.
3275  * @param[out] flow_id
3276  *   The flow ID connected to this flow.
3277  *
3278  * @return
3279  *   0 on success.
3280  */
3281 static int
3282 flow_hairpin_split(struct rte_eth_dev *dev,
3283                    const struct rte_flow_action actions[],
3284                    struct rte_flow_action actions_rx[],
3285                    struct rte_flow_action actions_tx[],
3286                    struct rte_flow_item pattern_tx[],
3287                    uint32_t *flow_id)
3288 {
3289         struct mlx5_priv *priv = dev->data->dev_private;
3290         const struct rte_flow_action_raw_encap *raw_encap;
3291         const struct rte_flow_action_raw_decap *raw_decap;
3292         struct mlx5_rte_flow_action_set_tag *set_tag;
3293         struct rte_flow_action *tag_action;
3294         struct mlx5_rte_flow_item_tag *tag_item;
3295         struct rte_flow_item *item;
3296         char *addr;
3297         int encap = 0;
3298
3299         mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
3300         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3301                 switch (actions->type) {
3302                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3303                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3304                         rte_memcpy(actions_tx, actions,
3305                                sizeof(struct rte_flow_action));
3306                         actions_tx++;
3307                         break;
3308                 case RTE_FLOW_ACTION_TYPE_COUNT:
3309                         if (encap) {
3310                                 rte_memcpy(actions_tx, actions,
3311                                            sizeof(struct rte_flow_action));
3312                                 actions_tx++;
3313                         } else {
3314                                 rte_memcpy(actions_rx, actions,
3315                                            sizeof(struct rte_flow_action));
3316                                 actions_rx++;
3317                         }
3318                         break;
3319                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3320                         raw_encap = actions->conf;
3321                         if (raw_encap->size >
3322                             (sizeof(struct rte_flow_item_eth) +
3323                              sizeof(struct rte_flow_item_ipv4))) {
3324                                 memcpy(actions_tx, actions,
3325                                        sizeof(struct rte_flow_action));
3326                                 actions_tx++;
3327                                 encap = 1;
3328                         } else {
3329                                 rte_memcpy(actions_rx, actions,
3330                                            sizeof(struct rte_flow_action));
3331                                 actions_rx++;
3332                         }
3333                         break;
3334                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3335                         raw_decap = actions->conf;
3336                         if (raw_decap->size <
3337                             (sizeof(struct rte_flow_item_eth) +
3338                              sizeof(struct rte_flow_item_ipv4))) {
3339                                 memcpy(actions_tx, actions,
3340                                        sizeof(struct rte_flow_action));
3341                                 actions_tx++;
3342                         } else {
3343                                 rte_memcpy(actions_rx, actions,
3344                                            sizeof(struct rte_flow_action));
3345                                 actions_rx++;
3346                         }
3347                         break;
3348                 default:
3349                         rte_memcpy(actions_rx, actions,
3350                                    sizeof(struct rte_flow_action));
3351                         actions_rx++;
3352                         break;
3353                 }
3354         }
3355         /* Add set meta action and end action for the Rx flow. */
3356         tag_action = actions_rx;
3357         tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3358         actions_rx++;
3359         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
3360         actions_rx++;
3361         set_tag = (void *)actions_rx;
3362         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
3363         assert(set_tag->id > REG_NONE);
3364         set_tag->data = *flow_id;
3365         tag_action->conf = set_tag;
3366         /* Create Tx item list. */
3367         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
3368         addr = (void *)&pattern_tx[2];
3369         item = pattern_tx;
3370         item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3371         tag_item = (void *)addr;
3372         tag_item->data = *flow_id;
3373         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
3374         assert(set_tag->id > REG_NONE);
3375         item->spec = tag_item;
3376         addr += sizeof(struct mlx5_rte_flow_item_tag);
3377         tag_item = (void *)addr;
3378         tag_item->data = UINT32_MAX;
3379         tag_item->id = UINT16_MAX;
3380         item->mask = tag_item;
3381         addr += sizeof(struct mlx5_rte_flow_item_tag);
3382         item->last = NULL;
3383         item++;
3384         item->type = RTE_FLOW_ITEM_TYPE_END;
3385         return 0;
3386 }
3387
3388 /**
3389  * The last stage of splitting chain, just creates the subflow
3390  * without any modification.
3391  *
3392  * @param dev
3393  *   Pointer to Ethernet device.
3394  * @param[in] flow
3395  *   Parent flow structure pointer.
3396  * @param[in, out] sub_flow
3397  *   Pointer to return the created subflow, may be NULL.
3398  * @param[in] attr
3399  *   Flow rule attributes.
3400  * @param[in] items
3401  *   Pattern specification (list terminated by the END pattern item).
3402  * @param[in] actions
3403  *   Associated actions (list terminated by the END action).
3404  * @param[in] external
3405  *   This flow rule is created by request external to PMD.
3406  * @param[out] error
3407  *   Perform verbose error reporting if not NULL.
3408  * @return
3409  *   0 on success, negative value otherwise
3410  */
3411 static int
3412 flow_create_split_inner(struct rte_eth_dev *dev,
3413                         struct rte_flow *flow,
3414                         struct mlx5_flow **sub_flow,
3415                         const struct rte_flow_attr *attr,
3416                         const struct rte_flow_item items[],
3417                         const struct rte_flow_action actions[],
3418                         bool external, struct rte_flow_error *error)
3419 {
3420         struct mlx5_flow *dev_flow;
3421
3422         dev_flow = flow_drv_prepare(flow, attr, items, actions, error);
3423         if (!dev_flow)
3424                 return -rte_errno;
3425         dev_flow->flow = flow;
3426         dev_flow->external = external;
3427         /* Subflow object was created, we must include one in the list. */
3428         LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
3429         if (sub_flow)
3430                 *sub_flow = dev_flow;
3431         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
3432 }
3433
3434 /**
3435  * Split the meter flow.
3436  *
3437  * As meter flow will split to three sub flow, other than meter
3438  * action, the other actions make sense to only meter accepts
3439  * the packet. If it need to be dropped, no other additional
3440  * actions should be take.
3441  *
3442  * One kind of special action which decapsulates the L3 tunnel
3443  * header will be in the prefix sub flow, as not to take the
3444  * L3 tunnel header into account.
3445  *
3446  * @param dev
3447  *   Pointer to Ethernet device.
3448  * @param[in] actions
3449  *   Associated actions (list terminated by the END action).
3450  * @param[out] actions_sfx
3451  *   Suffix flow actions.
3452  * @param[out] actions_pre
3453  *   Prefix flow actions.
3454  * @param[out] pattern_sfx
3455  *   The pattern items for the suffix flow.
3456  * @param[out] tag_sfx
3457  *   Pointer to suffix flow tag.
3458  *
3459  * @return
3460  *   0 on success.
3461  */
3462 static int
3463 flow_meter_split_prep(struct rte_eth_dev *dev,
3464                  const struct rte_flow_action actions[],
3465                  struct rte_flow_action actions_sfx[],
3466                  struct rte_flow_action actions_pre[])
3467 {
3468         struct rte_flow_action *tag_action;
3469         struct mlx5_rte_flow_action_set_tag *set_tag;
3470         struct rte_flow_error error;
3471         const struct rte_flow_action_raw_encap *raw_encap;
3472         const struct rte_flow_action_raw_decap *raw_decap;
3473         uint32_t tag_id;
3474
3475         /* Add the extra tag action first. */
3476         tag_action = actions_pre;
3477         tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3478         actions_pre++;
3479         /* Prepare the actions for prefix and suffix flow. */
3480         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3481                 switch (actions->type) {
3482                 case RTE_FLOW_ACTION_TYPE_METER:
3483                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3484                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3485                         memcpy(actions_pre, actions,
3486                                sizeof(struct rte_flow_action));
3487                         actions_pre++;
3488                         break;
3489                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3490                         raw_encap = actions->conf;
3491                         if (raw_encap->size >
3492                             (sizeof(struct rte_flow_item_eth) +
3493                              sizeof(struct rte_flow_item_ipv4))) {
3494                                 memcpy(actions_sfx, actions,
3495                                        sizeof(struct rte_flow_action));
3496                                 actions_sfx++;
3497                         } else {
3498                                 rte_memcpy(actions_pre, actions,
3499                                            sizeof(struct rte_flow_action));
3500                                 actions_pre++;
3501                         }
3502                         break;
3503                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3504                         raw_decap = actions->conf;
3505                         /* Size 0 decap means 50 bytes as vxlan decap. */
3506                         if (raw_decap->size && (raw_decap->size <
3507                             (sizeof(struct rte_flow_item_eth) +
3508                              sizeof(struct rte_flow_item_ipv4)))) {
3509                                 memcpy(actions_sfx, actions,
3510                                        sizeof(struct rte_flow_action));
3511                                 actions_sfx++;
3512                         } else {
3513                                 rte_memcpy(actions_pre, actions,
3514                                            sizeof(struct rte_flow_action));
3515                                 actions_pre++;
3516                         }
3517                         break;
3518                 default:
3519                         memcpy(actions_sfx, actions,
3520                                 sizeof(struct rte_flow_action));
3521                         actions_sfx++;
3522                         break;
3523                 }
3524         }
3525         /* Add end action to the actions. */
3526         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
3527         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
3528         actions_pre++;
3529         /* Set the tag. */
3530         set_tag = (void *)actions_pre;
3531         set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
3532         /*
3533          * Get the id from the qrss_pool to make qrss share the id with meter.
3534          */
3535         tag_id = flow_qrss_get_id(dev);
3536         set_tag->data = rte_cpu_to_be_32(tag_id);
3537         tag_action->conf = set_tag;
3538         return tag_id;
3539 }
3540
3541 /**
3542  * Split action list having QUEUE/RSS for metadata register copy.
3543  *
3544  * Once Q/RSS action is detected in user's action list, the flow action
3545  * should be split in order to copy metadata registers, which will happen in
3546  * RX_CP_TBL like,
3547  *   - CQE->flow_tag := reg_c[1] (MARK)
3548  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3549  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
3550  * This is because the last action of each flow must be a terminal action
3551  * (QUEUE, RSS or DROP).
3552  *
3553  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
3554  * stored and kept in the mlx5_flow structure per each sub_flow.
3555  *
3556  * The Q/RSS action is replaced with,
3557  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
3558  * And the following JUMP action is added at the end,
3559  *   - JUMP, to RX_CP_TBL.
3560  *
3561  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
3562  * flow_create_split_metadata() routine. The flow will look like,
3563  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
3564  *
3565  * @param dev
3566  *   Pointer to Ethernet device.
3567  * @param[out] split_actions
3568  *   Pointer to store split actions to jump to CP_TBL.
3569  * @param[in] actions
3570  *   Pointer to the list of original flow actions.
3571  * @param[in] qrss
3572  *   Pointer to the Q/RSS action.
3573  * @param[in] actions_n
3574  *   Number of original actions.
3575  * @param[out] error
3576  *   Perform verbose error reporting if not NULL.
3577  *
3578  * @return
3579  *   non-zero unique flow_id on success, otherwise 0 and
3580  *   error/rte_error are set.
3581  */
3582 static uint32_t
3583 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
3584                           struct rte_flow_action *split_actions,
3585                           const struct rte_flow_action *actions,
3586                           const struct rte_flow_action *qrss,
3587                           int actions_n, struct rte_flow_error *error)
3588 {
3589         struct mlx5_rte_flow_action_set_tag *set_tag;
3590         struct rte_flow_action_jump *jump;
3591         const int qrss_idx = qrss - actions;
3592         uint32_t flow_id = 0;
3593         int ret = 0;
3594
3595         /*
3596          * Given actions will be split
3597          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
3598          * - Add jump to mreg CP_TBL.
3599          * As a result, there will be one more action.
3600          */
3601         ++actions_n;
3602         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
3603         set_tag = (void *)(split_actions + actions_n);
3604         /*
3605          * If tag action is not set to void(it means we are not the meter
3606          * suffix flow), add the tag action. Since meter suffix flow already
3607          * has the tag added.
3608          */
3609         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
3610                 /*
3611                  * Allocate the new subflow ID. This one is unique within
3612                  * device and not shared with representors. Otherwise,
3613                  * we would have to resolve multi-thread access synch
3614                  * issue. Each flow on the shared device is appended
3615                  * with source vport identifier, so the resulting
3616                  * flows will be unique in the shared (by master and
3617                  * representors) domain even if they have coinciding
3618                  * IDs.
3619                  */
3620                 flow_id = flow_qrss_get_id(dev);
3621                 if (!flow_id)
3622                         return rte_flow_error_set(error, ENOMEM,
3623                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3624                                                   NULL, "can't allocate id "
3625                                                   "for split Q/RSS subflow");
3626                 /* Internal SET_TAG action to set flow ID. */
3627                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
3628                         .data = flow_id,
3629                 };
3630                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
3631                 if (ret < 0)
3632                         return ret;
3633                 set_tag->id = ret;
3634                 /* Construct new actions array. */
3635                 /* Replace QUEUE/RSS action. */
3636                 split_actions[qrss_idx] = (struct rte_flow_action){
3637                         .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG,
3638                         .conf = set_tag,
3639                 };
3640         }
3641         /* JUMP action to jump to mreg copy table (CP_TBL). */
3642         jump = (void *)(set_tag + 1);
3643         *jump = (struct rte_flow_action_jump){
3644                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3645         };
3646         split_actions[actions_n - 2] = (struct rte_flow_action){
3647                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3648                 .conf = jump,
3649         };
3650         split_actions[actions_n - 1] = (struct rte_flow_action){
3651                 .type = RTE_FLOW_ACTION_TYPE_END,
3652         };
3653         return flow_id;
3654 }
3655
3656 /**
3657  * Extend the given action list for Tx metadata copy.
3658  *
3659  * Copy the given action list to the ext_actions and add flow metadata register
3660  * copy action in order to copy reg_a set by WQE to reg_c[0].
3661  *
3662  * @param[out] ext_actions
3663  *   Pointer to the extended action list.
3664  * @param[in] actions
3665  *   Pointer to the list of actions.
3666  * @param[in] actions_n
3667  *   Number of actions in the list.
3668  * @param[out] error
3669  *   Perform verbose error reporting if not NULL.
3670  *
3671  * @return
3672  *   0 on success, negative value otherwise
3673  */
3674 static int
3675 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
3676                        struct rte_flow_action *ext_actions,
3677                        const struct rte_flow_action *actions,
3678                        int actions_n, struct rte_flow_error *error)
3679 {
3680         struct mlx5_flow_action_copy_mreg *cp_mreg =
3681                 (struct mlx5_flow_action_copy_mreg *)
3682                         (ext_actions + actions_n + 1);
3683         int ret;
3684
3685         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3686         if (ret < 0)
3687                 return ret;
3688         cp_mreg->dst = ret;
3689         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
3690         if (ret < 0)
3691                 return ret;
3692         cp_mreg->src = ret;
3693         memcpy(ext_actions, actions,
3694                         sizeof(*ext_actions) * actions_n);
3695         ext_actions[actions_n - 1] = (struct rte_flow_action){
3696                 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3697                 .conf = cp_mreg,
3698         };
3699         ext_actions[actions_n] = (struct rte_flow_action){
3700                 .type = RTE_FLOW_ACTION_TYPE_END,
3701         };
3702         return 0;
3703 }
3704
3705 /**
3706  * The splitting for metadata feature.
3707  *
3708  * - Q/RSS action on NIC Rx should be split in order to pass by
3709  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
3710  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
3711  *
3712  * - All the actions on NIC Tx should have a mreg copy action to
3713  *   copy reg_a from WQE to reg_c[0].
3714  *
3715  * @param dev
3716  *   Pointer to Ethernet device.
3717  * @param[in] flow
3718  *   Parent flow structure pointer.
3719  * @param[in] attr
3720  *   Flow rule attributes.
3721  * @param[in] items
3722  *   Pattern specification (list terminated by the END pattern item).
3723  * @param[in] actions
3724  *   Associated actions (list terminated by the END action).
3725  * @param[in] external
3726  *   This flow rule is created by request external to PMD.
3727  * @param[out] error
3728  *   Perform verbose error reporting if not NULL.
3729  * @return
3730  *   0 on success, negative value otherwise
3731  */
3732 static int
3733 flow_create_split_metadata(struct rte_eth_dev *dev,
3734                            struct rte_flow *flow,
3735                            const struct rte_flow_attr *attr,
3736                            const struct rte_flow_item items[],
3737                            const struct rte_flow_action actions[],
3738                            bool external, struct rte_flow_error *error)
3739 {
3740         struct mlx5_priv *priv = dev->data->dev_private;
3741         struct mlx5_dev_config *config = &priv->config;
3742         const struct rte_flow_action *qrss = NULL;
3743         struct rte_flow_action *ext_actions = NULL;
3744         struct mlx5_flow *dev_flow = NULL;
3745         uint32_t qrss_id = 0;
3746         int mtr_sfx = 0;
3747         size_t act_size;
3748         int actions_n;
3749         int ret;
3750
3751         /* Check whether extensive metadata feature is engaged. */
3752         if (!config->dv_flow_en ||
3753             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3754             !mlx5_flow_ext_mreg_supported(dev))
3755                 return flow_create_split_inner(dev, flow, NULL, attr, items,
3756                                                actions, external, error);
3757         actions_n = flow_parse_qrss_action(actions, &qrss);
3758         if (qrss) {
3759                 /* Exclude hairpin flows from splitting. */
3760                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
3761                         const struct rte_flow_action_queue *queue;
3762
3763                         queue = qrss->conf;
3764                         if (mlx5_rxq_get_type(dev, queue->index) ==
3765                             MLX5_RXQ_TYPE_HAIRPIN)
3766                                 qrss = NULL;
3767                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
3768                         const struct rte_flow_action_rss *rss;
3769
3770                         rss = qrss->conf;
3771                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
3772                             MLX5_RXQ_TYPE_HAIRPIN)
3773                                 qrss = NULL;
3774                 }
3775         }
3776         if (qrss) {
3777                 /* Check if it is in meter suffix table. */
3778                 mtr_sfx = attr->group == (attr->transfer ?
3779                           (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
3780                           MLX5_FLOW_TABLE_LEVEL_SUFFIX);
3781                 /*
3782                  * Q/RSS action on NIC Rx should be split in order to pass by
3783                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
3784                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
3785                  */
3786                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
3787                            sizeof(struct rte_flow_action_set_tag) +
3788                            sizeof(struct rte_flow_action_jump);
3789                 ext_actions = rte_zmalloc(__func__, act_size, 0);
3790                 if (!ext_actions)
3791                         return rte_flow_error_set(error, ENOMEM,
3792                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3793                                                   NULL, "no memory to split "
3794                                                   "metadata flow");
3795                 /*
3796                  * If we are the suffix flow of meter, tag already exist.
3797                  * Set the tag action to void.
3798                  */
3799                 if (mtr_sfx)
3800                         ext_actions[qrss - actions].type =
3801                                                 RTE_FLOW_ACTION_TYPE_VOID;
3802                 else
3803                         ext_actions[qrss - actions].type =
3804                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3805                 /*
3806                  * Create the new actions list with removed Q/RSS action
3807                  * and appended set tag and jump to register copy table
3808                  * (RX_CP_TBL). We should preallocate unique tag ID here
3809                  * in advance, because it is needed for set tag action.
3810                  */
3811                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
3812                                                     qrss, actions_n, error);
3813                 if (!mtr_sfx && !qrss_id) {
3814                         ret = -rte_errno;
3815                         goto exit;
3816                 }
3817         } else if (attr->egress && !attr->transfer) {
3818                 /*
3819                  * All the actions on NIC Tx should have a metadata register
3820                  * copy action to copy reg_a from WQE to reg_c[meta]
3821                  */
3822                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
3823                            sizeof(struct mlx5_flow_action_copy_mreg);
3824                 ext_actions = rte_zmalloc(__func__, act_size, 0);
3825                 if (!ext_actions)
3826                         return rte_flow_error_set(error, ENOMEM,
3827                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3828                                                   NULL, "no memory to split "
3829                                                   "metadata flow");
3830                 /* Create the action list appended with copy register. */
3831                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
3832                                              actions_n, error);
3833                 if (ret < 0)
3834                         goto exit;
3835         }
3836         /* Add the unmodified original or prefix subflow. */
3837         ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items,
3838                                       ext_actions ? ext_actions : actions,
3839                                       external, error);
3840         if (ret < 0)
3841                 goto exit;
3842         assert(dev_flow);
3843         if (qrss) {
3844                 const struct rte_flow_attr q_attr = {
3845                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3846                         .ingress = 1,
3847                 };
3848                 /* Internal PMD action to set register. */
3849                 struct mlx5_rte_flow_item_tag q_tag_spec = {
3850                         .data = qrss_id,
3851                         .id = 0,
3852                 };
3853                 struct rte_flow_item q_items[] = {
3854                         {
3855                                 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3856                                 .spec = &q_tag_spec,
3857                                 .last = NULL,
3858                                 .mask = NULL,
3859                         },
3860                         {
3861                                 .type = RTE_FLOW_ITEM_TYPE_END,
3862                         },
3863                 };
3864                 struct rte_flow_action q_actions[] = {
3865                         {
3866                                 .type = qrss->type,
3867                                 .conf = qrss->conf,
3868                         },
3869                         {
3870                                 .type = RTE_FLOW_ACTION_TYPE_END,
3871                         },
3872                 };
3873                 uint64_t hash_fields = dev_flow->hash_fields;
3874
3875                 /*
3876                  * Configure the tag item only if there is no meter subflow.
3877                  * Since tag is already marked in the meter suffix subflow
3878                  * we can just use the meter suffix items as is.
3879                  */
3880                 if (qrss_id) {
3881                         /* Not meter subflow. */
3882                         assert(!mtr_sfx);
3883                         /*
3884                          * Put unique id in prefix flow due to it is destroyed
3885                          * after suffix flow and id will be freed after there
3886                          * is no actual flows with this id and identifier
3887                          * reallocation becomes possible (for example, for
3888                          * other flows in other threads).
3889                          */
3890                         dev_flow->qrss_id = qrss_id;
3891                         qrss_id = 0;
3892                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
3893                                                    error);
3894                         if (ret < 0)
3895                                 goto exit;
3896                         q_tag_spec.id = ret;
3897                 }
3898                 dev_flow = NULL;
3899                 /* Add suffix subflow to execute Q/RSS. */
3900                 ret = flow_create_split_inner(dev, flow, &dev_flow,
3901                                               &q_attr, mtr_sfx ? items :
3902                                               q_items, q_actions,
3903                                               external, error);
3904                 if (ret < 0)
3905                         goto exit;
3906                 assert(dev_flow);
3907                 dev_flow->hash_fields = hash_fields;
3908         }
3909
3910 exit:
3911         /*
3912          * We do not destroy the partially created sub_flows in case of error.
3913          * These ones are included into parent flow list and will be destroyed
3914          * by flow_drv_destroy.
3915          */
3916         flow_qrss_free_id(dev, qrss_id);
3917         rte_free(ext_actions);
3918         return ret;
3919 }
3920
3921 /**
3922  * The splitting for meter feature.
3923  *
3924  * - The meter flow will be split to two flows as prefix and
3925  *   suffix flow. The packets make sense only it pass the prefix
3926  *   meter action.
3927  *
3928  * - Reg_C_5 is used for the packet to match betweend prefix and
3929  *   suffix flow.
3930  *
3931  * @param dev
3932  *   Pointer to Ethernet device.
3933  * @param[in] flow
3934  *   Parent flow structure pointer.
3935  * @param[in] attr
3936  *   Flow rule attributes.
3937  * @param[in] items
3938  *   Pattern specification (list terminated by the END pattern item).
3939  * @param[in] actions
3940  *   Associated actions (list terminated by the END action).
3941  * @param[in] external
3942  *   This flow rule is created by request external to PMD.
3943  * @param[out] error
3944  *   Perform verbose error reporting if not NULL.
3945  * @return
3946  *   0 on success, negative value otherwise
3947  */
3948 static int
3949 flow_create_split_meter(struct rte_eth_dev *dev,
3950                            struct rte_flow *flow,
3951                            const struct rte_flow_attr *attr,
3952                            const struct rte_flow_item items[],
3953                            const struct rte_flow_action actions[],
3954                            bool external, struct rte_flow_error *error)
3955 {
3956         struct mlx5_priv *priv = dev->data->dev_private;
3957         struct rte_flow_action *sfx_actions = NULL;
3958         struct rte_flow_action *pre_actions = NULL;
3959         struct rte_flow_item *sfx_items = NULL;
3960         const  struct rte_flow_item *sfx_port_id_item;
3961         struct mlx5_flow *dev_flow = NULL;
3962         struct rte_flow_attr sfx_attr = *attr;
3963         uint32_t mtr = 0;
3964         uint32_t mtr_tag_id = 0;
3965         size_t act_size;
3966         size_t item_size;
3967         int actions_n = 0;
3968         int ret;
3969
3970         if (priv->mtr_en)
3971                 actions_n = flow_check_meter_action(actions, &mtr);
3972         if (mtr) {
3973                 struct mlx5_rte_flow_item_tag *tag_spec;
3974                 /* The five prefix actions: meter, decap, encap, tag, end. */
3975                 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
3976                            sizeof(struct rte_flow_action_set_tag);
3977                 /* tag, end. */
3978 #define METER_SUFFIX_ITEM 3
3979                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
3980                             sizeof(struct mlx5_rte_flow_item_tag);
3981                 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0);
3982                 if (!sfx_actions)
3983                         return rte_flow_error_set(error, ENOMEM,
3984                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3985                                                   NULL, "no memory to split "
3986                                                   "meter flow");
3987                 pre_actions = sfx_actions + actions_n;
3988                 mtr_tag_id = flow_meter_split_prep(dev, actions, sfx_actions,
3989                                                      pre_actions);
3990                 if (!mtr_tag_id) {
3991                         ret = -rte_errno;
3992                         goto exit;
3993                 }
3994                 /* Add the prefix subflow. */
3995                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items,
3996                                                   pre_actions, external, error);
3997                 if (ret) {
3998                         ret = -rte_errno;
3999                         goto exit;
4000                 }
4001                 dev_flow->mtr_flow_id = mtr_tag_id;
4002                 /* Prepare the suffix flow match pattern. */
4003                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4004                              act_size);
4005                 tag_spec = (struct mlx5_rte_flow_item_tag *)(sfx_items +
4006                             METER_SUFFIX_ITEM);
4007                 tag_spec->data = rte_cpu_to_be_32(dev_flow->mtr_flow_id);
4008                 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0,
4009                                                     error);
4010                 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4011                 sfx_items->spec = tag_spec;
4012                 sfx_items->last = NULL;
4013                 sfx_items->mask = NULL;
4014                 sfx_items++;
4015                 sfx_port_id_item = find_port_id_item(items);
4016                 if (sfx_port_id_item) {
4017                         memcpy(sfx_items, sfx_port_id_item,
4018                                sizeof(*sfx_items));
4019                         sfx_items++;
4020                 }
4021                 sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4022                 sfx_items -= sfx_port_id_item ? 2 : 1;
4023                 /* Setting the sfx group atrr. */
4024                 sfx_attr.group = sfx_attr.transfer ?
4025                                 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4026                                  MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4027         }
4028         /* Add the prefix subflow. */
4029         ret = flow_create_split_metadata(dev, flow, &sfx_attr,
4030                                          sfx_items ? sfx_items : items,
4031                                          sfx_actions ? sfx_actions : actions,
4032                                          external, error);
4033 exit:
4034         if (sfx_actions)
4035                 rte_free(sfx_actions);
4036         return ret;
4037 }
4038
4039 /**
4040  * Split the flow to subflow set. The splitters might be linked
4041  * in the chain, like this:
4042  * flow_create_split_outer() calls:
4043  *   flow_create_split_meter() calls:
4044  *     flow_create_split_metadata(meter_subflow_0) calls:
4045  *       flow_create_split_inner(metadata_subflow_0)
4046  *       flow_create_split_inner(metadata_subflow_1)
4047  *       flow_create_split_inner(metadata_subflow_2)
4048  *     flow_create_split_metadata(meter_subflow_1) calls:
4049  *       flow_create_split_inner(metadata_subflow_0)
4050  *       flow_create_split_inner(metadata_subflow_1)
4051  *       flow_create_split_inner(metadata_subflow_2)
4052  *
4053  * This provide flexible way to add new levels of flow splitting.
4054  * The all of successfully created subflows are included to the
4055  * parent flow dev_flow list.
4056  *
4057  * @param dev
4058  *   Pointer to Ethernet device.
4059  * @param[in] flow
4060  *   Parent flow structure pointer.
4061  * @param[in] attr
4062  *   Flow rule attributes.
4063  * @param[in] items
4064  *   Pattern specification (list terminated by the END pattern item).
4065  * @param[in] actions
4066  *   Associated actions (list terminated by the END action).
4067  * @param[in] external
4068  *   This flow rule is created by request external to PMD.
4069  * @param[out] error
4070  *   Perform verbose error reporting if not NULL.
4071  * @return
4072  *   0 on success, negative value otherwise
4073  */
4074 static int
4075 flow_create_split_outer(struct rte_eth_dev *dev,
4076                         struct rte_flow *flow,
4077                         const struct rte_flow_attr *attr,
4078                         const struct rte_flow_item items[],
4079                         const struct rte_flow_action actions[],
4080                         bool external, struct rte_flow_error *error)
4081 {
4082         int ret;
4083
4084         ret = flow_create_split_meter(dev, flow, attr, items,
4085                                          actions, external, error);
4086         assert(ret <= 0);
4087         return ret;
4088 }
4089
4090 /**
4091  * Create a flow and add it to @p list.
4092  *
4093  * @param dev
4094  *   Pointer to Ethernet device.
4095  * @param list
4096  *   Pointer to a TAILQ flow list. If this parameter NULL,
4097  *   no list insertion occurred, flow is just created,
4098  *   this is caller's responsibility to track the
4099  *   created flow.
4100  * @param[in] attr
4101  *   Flow rule attributes.
4102  * @param[in] items
4103  *   Pattern specification (list terminated by the END pattern item).
4104  * @param[in] actions
4105  *   Associated actions (list terminated by the END action).
4106  * @param[in] external
4107  *   This flow rule is created by request external to PMD.
4108  * @param[out] error
4109  *   Perform verbose error reporting if not NULL.
4110  *
4111  * @return
4112  *   A flow on success, NULL otherwise and rte_errno is set.
4113  */
4114 static struct rte_flow *
4115 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
4116                  const struct rte_flow_attr *attr,
4117                  const struct rte_flow_item items[],
4118                  const struct rte_flow_action actions[],
4119                  bool external, struct rte_flow_error *error)
4120 {
4121         struct mlx5_priv *priv = dev->data->dev_private;
4122         struct rte_flow *flow = NULL;
4123         struct mlx5_flow *dev_flow;
4124         const struct rte_flow_action_rss *rss;
4125         union {
4126                 struct rte_flow_expand_rss buf;
4127                 uint8_t buffer[2048];
4128         } expand_buffer;
4129         union {
4130                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4131                 uint8_t buffer[2048];
4132         } actions_rx;
4133         union {
4134                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4135                 uint8_t buffer[2048];
4136         } actions_hairpin_tx;
4137         union {
4138                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
4139                 uint8_t buffer[2048];
4140         } items_tx;
4141         struct rte_flow_expand_rss *buf = &expand_buffer.buf;
4142         const struct rte_flow_action *p_actions_rx = actions;
4143         int ret;
4144         uint32_t i;
4145         uint32_t flow_size;
4146         int hairpin_flow = 0;
4147         uint32_t hairpin_id = 0;
4148         struct rte_flow_attr attr_tx = { .priority = 0 };
4149
4150         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4151         if (hairpin_flow > 0) {
4152                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
4153                         rte_errno = EINVAL;
4154                         return NULL;
4155                 }
4156                 flow_hairpin_split(dev, actions, actions_rx.actions,
4157                                    actions_hairpin_tx.actions, items_tx.items,
4158                                    &hairpin_id);
4159                 p_actions_rx = actions_rx.actions;
4160         }
4161         ret = flow_drv_validate(dev, attr, items, p_actions_rx, external,
4162                                 error);
4163         if (ret < 0)
4164                 goto error_before_flow;
4165         flow_size = sizeof(struct rte_flow);
4166         rss = flow_get_rss_action(p_actions_rx);
4167         if (rss)
4168                 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
4169                                             sizeof(void *));
4170         else
4171                 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
4172         flow = rte_calloc(__func__, 1, flow_size, 0);
4173         if (!flow) {
4174                 rte_errno = ENOMEM;
4175                 goto error_before_flow;
4176         }
4177         flow->drv_type = flow_get_drv_type(dev, attr);
4178         if (hairpin_id != 0)
4179                 flow->hairpin_flow_id = hairpin_id;
4180         assert(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
4181                flow->drv_type < MLX5_FLOW_TYPE_MAX);
4182         flow->rss.queue = (void *)(flow + 1);
4183         if (rss) {
4184                 /*
4185                  * The following information is required by
4186                  * mlx5_flow_hashfields_adjust() in advance.
4187                  */
4188                 flow->rss.level = rss->level;
4189                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
4190                 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
4191         }
4192         LIST_INIT(&flow->dev_flows);
4193         if (rss && rss->types) {
4194                 unsigned int graph_root;
4195
4196                 graph_root = find_graph_root(items, rss->level);
4197                 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
4198                                           items, rss->types,
4199                                           mlx5_support_expansion,
4200                                           graph_root);
4201                 assert(ret > 0 &&
4202                        (unsigned int)ret < sizeof(expand_buffer.buffer));
4203         } else {
4204                 buf->entries = 1;
4205                 buf->entry[0].pattern = (void *)(uintptr_t)items;
4206         }
4207         for (i = 0; i < buf->entries; ++i) {
4208                 /*
4209                  * The splitter may create multiple dev_flows,
4210                  * depending on configuration. In the simplest
4211                  * case it just creates unmodified original flow.
4212                  */
4213                 ret = flow_create_split_outer(dev, flow, attr,
4214                                               buf->entry[i].pattern,
4215                                               p_actions_rx, external,
4216                                               error);
4217                 if (ret < 0)
4218                         goto error;
4219         }
4220         /* Create the tx flow. */
4221         if (hairpin_flow) {
4222                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
4223                 attr_tx.ingress = 0;
4224                 attr_tx.egress = 1;
4225                 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items,
4226                                             actions_hairpin_tx.actions, error);
4227                 if (!dev_flow)
4228                         goto error;
4229                 dev_flow->flow = flow;
4230                 dev_flow->external = 0;
4231                 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
4232                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
4233                                          items_tx.items,
4234                                          actions_hairpin_tx.actions, error);
4235                 if (ret < 0)
4236                         goto error;
4237         }
4238         /*
4239          * Update the metadata register copy table. If extensive
4240          * metadata feature is enabled and registers are supported
4241          * we might create the extra rte_flow for each unique
4242          * MARK/FLAG action ID.
4243          *
4244          * The table is updated for ingress Flows only, because
4245          * the egress Flows belong to the different device and
4246          * copy table should be updated in peer NIC Rx domain.
4247          */
4248         if (attr->ingress &&
4249             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
4250                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
4251                 if (ret)
4252                         goto error;
4253         }
4254         if (dev->data->dev_started) {
4255                 ret = flow_drv_apply(dev, flow, error);
4256                 if (ret < 0)
4257                         goto error;
4258         }
4259         if (list)
4260                 TAILQ_INSERT_TAIL(list, flow, next);
4261         flow_rxq_flags_set(dev, flow);
4262         return flow;
4263 error_before_flow:
4264         if (hairpin_id)
4265                 mlx5_flow_id_release(priv->sh->flow_id_pool,
4266                                      hairpin_id);
4267         return NULL;
4268 error:
4269         assert(flow);
4270         flow_mreg_del_copy_action(dev, flow);
4271         ret = rte_errno; /* Save rte_errno before cleanup. */
4272         if (flow->hairpin_flow_id)
4273                 mlx5_flow_id_release(priv->sh->flow_id_pool,
4274                                      flow->hairpin_flow_id);
4275         assert(flow);
4276         flow_drv_destroy(dev, flow);
4277         rte_free(flow);
4278         rte_errno = ret; /* Restore rte_errno. */
4279         return NULL;
4280 }
4281
4282 /**
4283  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
4284  * incoming packets to table 1.
4285  *
4286  * Other flow rules, requested for group n, will be created in
4287  * e-switch table n+1.
4288  * Jump action to e-switch group n will be created to group n+1.
4289  *
4290  * Used when working in switchdev mode, to utilise advantages of table 1
4291  * and above.
4292  *
4293  * @param dev
4294  *   Pointer to Ethernet device.
4295  *
4296  * @return
4297  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
4298  */
4299 struct rte_flow *
4300 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
4301 {
4302         const struct rte_flow_attr attr = {
4303                 .group = 0,
4304                 .priority = 0,
4305                 .ingress = 1,
4306                 .egress = 0,
4307                 .transfer = 1,
4308         };
4309         const struct rte_flow_item pattern = {
4310                 .type = RTE_FLOW_ITEM_TYPE_END,
4311         };
4312         struct rte_flow_action_jump jump = {
4313                 .group = 1,
4314         };
4315         const struct rte_flow_action actions[] = {
4316                 {
4317                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4318                         .conf = &jump,
4319                 },
4320                 {
4321                         .type = RTE_FLOW_ACTION_TYPE_END,
4322                 },
4323         };
4324         struct mlx5_priv *priv = dev->data->dev_private;
4325         struct rte_flow_error error;
4326
4327         return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern,
4328                                 actions, false, &error);
4329 }
4330
4331 /**
4332  * Create a flow.
4333  *
4334  * @see rte_flow_create()
4335  * @see rte_flow_ops
4336  */
4337 struct rte_flow *
4338 mlx5_flow_create(struct rte_eth_dev *dev,
4339                  const struct rte_flow_attr *attr,
4340                  const struct rte_flow_item items[],
4341                  const struct rte_flow_action actions[],
4342                  struct rte_flow_error *error)
4343 {
4344         struct mlx5_priv *priv = dev->data->dev_private;
4345
4346         return flow_list_create(dev, &priv->flows,
4347                                 attr, items, actions, true, error);
4348 }
4349
4350 /**
4351  * Destroy a flow in a list.
4352  *
4353  * @param dev
4354  *   Pointer to Ethernet device.
4355  * @param list
4356  *   Pointer to a TAILQ flow list. If this parameter NULL,
4357  *   there is no flow removal from the list.
4358  * @param[in] flow
4359  *   Flow to destroy.
4360  */
4361 static void
4362 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
4363                   struct rte_flow *flow)
4364 {
4365         struct mlx5_priv *priv = dev->data->dev_private;
4366
4367         /*
4368          * Update RX queue flags only if port is started, otherwise it is
4369          * already clean.
4370          */
4371         if (dev->data->dev_started)
4372                 flow_rxq_flags_trim(dev, flow);
4373         if (flow->hairpin_flow_id)
4374                 mlx5_flow_id_release(priv->sh->flow_id_pool,
4375                                      flow->hairpin_flow_id);
4376         flow_drv_destroy(dev, flow);
4377         if (list)
4378                 TAILQ_REMOVE(list, flow, next);
4379         flow_mreg_del_copy_action(dev, flow);
4380         rte_free(flow->fdir);
4381         rte_free(flow);
4382 }
4383
4384 /**
4385  * Destroy all flows.
4386  *
4387  * @param dev
4388  *   Pointer to Ethernet device.
4389  * @param list
4390  *   Pointer to a TAILQ flow list.
4391  */
4392 void
4393 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
4394 {
4395         while (!TAILQ_EMPTY(list)) {
4396                 struct rte_flow *flow;
4397
4398                 flow = TAILQ_FIRST(list);
4399                 flow_list_destroy(dev, list, flow);
4400         }
4401 }
4402
4403 /**
4404  * Remove all flows.
4405  *
4406  * @param dev
4407  *   Pointer to Ethernet device.
4408  * @param list
4409  *   Pointer to a TAILQ flow list.
4410  */
4411 void
4412 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
4413 {
4414         struct rte_flow *flow;
4415
4416         TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
4417                 flow_drv_remove(dev, flow);
4418                 flow_mreg_stop_copy_action(dev, flow);
4419         }
4420         flow_mreg_del_default_copy_action(dev);
4421         flow_rxq_flags_clear(dev);
4422 }
4423
4424 /**
4425  * Add all flows.
4426  *
4427  * @param dev
4428  *   Pointer to Ethernet device.
4429  * @param list
4430  *   Pointer to a TAILQ flow list.
4431  *
4432  * @return
4433  *   0 on success, a negative errno value otherwise and rte_errno is set.
4434  */
4435 int
4436 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
4437 {
4438         struct rte_flow *flow;
4439         struct rte_flow_error error;
4440         int ret = 0;
4441
4442         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
4443         ret = flow_mreg_add_default_copy_action(dev, &error);
4444         if (ret < 0)
4445                 return -rte_errno;
4446         /* Apply Flows created by application. */
4447         TAILQ_FOREACH(flow, list, next) {
4448                 ret = flow_mreg_start_copy_action(dev, flow);
4449                 if (ret < 0)
4450                         goto error;
4451                 ret = flow_drv_apply(dev, flow, &error);
4452                 if (ret < 0)
4453                         goto error;
4454                 flow_rxq_flags_set(dev, flow);
4455         }
4456         return 0;
4457 error:
4458         ret = rte_errno; /* Save rte_errno before cleanup. */
4459         mlx5_flow_stop(dev, list);
4460         rte_errno = ret; /* Restore rte_errno. */
4461         return -rte_errno;
4462 }
4463
4464 /**
4465  * Verify the flow list is empty
4466  *
4467  * @param dev
4468  *  Pointer to Ethernet device.
4469  *
4470  * @return the number of flows not released.
4471  */
4472 int
4473 mlx5_flow_verify(struct rte_eth_dev *dev)
4474 {
4475         struct mlx5_priv *priv = dev->data->dev_private;
4476         struct rte_flow *flow;
4477         int ret = 0;
4478
4479         TAILQ_FOREACH(flow, &priv->flows, next) {
4480                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
4481                         dev->data->port_id, (void *)flow);
4482                 ++ret;
4483         }
4484         return ret;
4485 }
4486
4487 /**
4488  * Enable default hairpin egress flow.
4489  *
4490  * @param dev
4491  *   Pointer to Ethernet device.
4492  * @param queue
4493  *   The queue index.
4494  *
4495  * @return
4496  *   0 on success, a negative errno value otherwise and rte_errno is set.
4497  */
4498 int
4499 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
4500                             uint32_t queue)
4501 {
4502         struct mlx5_priv *priv = dev->data->dev_private;
4503         const struct rte_flow_attr attr = {
4504                 .egress = 1,
4505                 .priority = 0,
4506         };
4507         struct mlx5_rte_flow_item_tx_queue queue_spec = {
4508                 .queue = queue,
4509         };
4510         struct mlx5_rte_flow_item_tx_queue queue_mask = {
4511                 .queue = UINT32_MAX,
4512         };
4513         struct rte_flow_item items[] = {
4514                 {
4515                         .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
4516                         .spec = &queue_spec,
4517                         .last = NULL,
4518                         .mask = &queue_mask,
4519                 },
4520                 {
4521                         .type = RTE_FLOW_ITEM_TYPE_END,
4522                 },
4523         };
4524         struct rte_flow_action_jump jump = {
4525                 .group = MLX5_HAIRPIN_TX_TABLE,
4526         };
4527         struct rte_flow_action actions[2];
4528         struct rte_flow *flow;
4529         struct rte_flow_error error;
4530
4531         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
4532         actions[0].conf = &jump;
4533         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
4534         flow = flow_list_create(dev, &priv->ctrl_flows,
4535                                 &attr, items, actions, false, &error);
4536         if (!flow) {
4537                 DRV_LOG(DEBUG,
4538                         "Failed to create ctrl flow: rte_errno(%d),"
4539                         " type(%d), message(%s)",
4540                         rte_errno, error.type,
4541                         error.message ? error.message : " (no stated reason)");
4542                 return -rte_errno;
4543         }
4544         return 0;
4545 }
4546
4547 /**
4548  * Enable a control flow configured from the control plane.
4549  *
4550  * @param dev
4551  *   Pointer to Ethernet device.
4552  * @param eth_spec
4553  *   An Ethernet flow spec to apply.
4554  * @param eth_mask
4555  *   An Ethernet flow mask to apply.
4556  * @param vlan_spec
4557  *   A VLAN flow spec to apply.
4558  * @param vlan_mask
4559  *   A VLAN flow mask to apply.
4560  *
4561  * @return
4562  *   0 on success, a negative errno value otherwise and rte_errno is set.
4563  */
4564 int
4565 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
4566                     struct rte_flow_item_eth *eth_spec,
4567                     struct rte_flow_item_eth *eth_mask,
4568                     struct rte_flow_item_vlan *vlan_spec,
4569                     struct rte_flow_item_vlan *vlan_mask)
4570 {
4571         struct mlx5_priv *priv = dev->data->dev_private;
4572         const struct rte_flow_attr attr = {
4573                 .ingress = 1,
4574                 .priority = MLX5_FLOW_PRIO_RSVD,
4575         };
4576         struct rte_flow_item items[] = {
4577                 {
4578                         .type = RTE_FLOW_ITEM_TYPE_ETH,
4579                         .spec = eth_spec,
4580                         .last = NULL,
4581                         .mask = eth_mask,
4582                 },
4583                 {
4584                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
4585                                               RTE_FLOW_ITEM_TYPE_END,
4586                         .spec = vlan_spec,
4587                         .last = NULL,
4588                         .mask = vlan_mask,
4589                 },
4590                 {
4591                         .type = RTE_FLOW_ITEM_TYPE_END,
4592                 },
4593         };
4594         uint16_t queue[priv->reta_idx_n];
4595         struct rte_flow_action_rss action_rss = {
4596                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
4597                 .level = 0,
4598                 .types = priv->rss_conf.rss_hf,
4599                 .key_len = priv->rss_conf.rss_key_len,
4600                 .queue_num = priv->reta_idx_n,
4601                 .key = priv->rss_conf.rss_key,
4602                 .queue = queue,
4603         };
4604         struct rte_flow_action actions[] = {
4605                 {
4606                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4607                         .conf = &action_rss,
4608                 },
4609                 {
4610                         .type = RTE_FLOW_ACTION_TYPE_END,
4611                 },
4612         };
4613         struct rte_flow *flow;
4614         struct rte_flow_error error;
4615         unsigned int i;
4616
4617         if (!priv->reta_idx_n || !priv->rxqs_n) {
4618                 return 0;
4619         }
4620         for (i = 0; i != priv->reta_idx_n; ++i)
4621                 queue[i] = (*priv->reta_idx)[i];
4622         flow = flow_list_create(dev, &priv->ctrl_flows,
4623                                 &attr, items, actions, false, &error);
4624         if (!flow)
4625                 return -rte_errno;
4626         return 0;
4627 }
4628
4629 /**
4630  * Enable a flow control configured from the control plane.
4631  *
4632  * @param dev
4633  *   Pointer to Ethernet device.
4634  * @param eth_spec
4635  *   An Ethernet flow spec to apply.
4636  * @param eth_mask
4637  *   An Ethernet flow mask to apply.
4638  *
4639  * @return
4640  *   0 on success, a negative errno value otherwise and rte_errno is set.
4641  */
4642 int
4643 mlx5_ctrl_flow(struct rte_eth_dev *dev,
4644                struct rte_flow_item_eth *eth_spec,
4645                struct rte_flow_item_eth *eth_mask)
4646 {
4647         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
4648 }
4649
4650 /**
4651  * Destroy a flow.
4652  *
4653  * @see rte_flow_destroy()
4654  * @see rte_flow_ops
4655  */
4656 int
4657 mlx5_flow_destroy(struct rte_eth_dev *dev,
4658                   struct rte_flow *flow,
4659                   struct rte_flow_error *error __rte_unused)
4660 {
4661         struct mlx5_priv *priv = dev->data->dev_private;
4662
4663         flow_list_destroy(dev, &priv->flows, flow);
4664         return 0;
4665 }
4666
4667 /**
4668  * Destroy all flows.
4669  *
4670  * @see rte_flow_flush()
4671  * @see rte_flow_ops
4672  */
4673 int
4674 mlx5_flow_flush(struct rte_eth_dev *dev,
4675                 struct rte_flow_error *error __rte_unused)
4676 {
4677         struct mlx5_priv *priv = dev->data->dev_private;
4678
4679         mlx5_flow_list_flush(dev, &priv->flows);
4680         return 0;
4681 }
4682
4683 /**
4684  * Isolated mode.
4685  *
4686  * @see rte_flow_isolate()
4687  * @see rte_flow_ops
4688  */
4689 int
4690 mlx5_flow_isolate(struct rte_eth_dev *dev,
4691                   int enable,
4692                   struct rte_flow_error *error)
4693 {
4694         struct mlx5_priv *priv = dev->data->dev_private;
4695
4696         if (dev->data->dev_started) {
4697                 rte_flow_error_set(error, EBUSY,
4698                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4699                                    NULL,
4700                                    "port must be stopped first");
4701                 return -rte_errno;
4702         }
4703         priv->isolated = !!enable;
4704         if (enable)
4705                 dev->dev_ops = &mlx5_dev_ops_isolate;
4706         else
4707                 dev->dev_ops = &mlx5_dev_ops;
4708         return 0;
4709 }
4710
4711 /**
4712  * Query a flow.
4713  *
4714  * @see rte_flow_query()
4715  * @see rte_flow_ops
4716  */
4717 static int
4718 flow_drv_query(struct rte_eth_dev *dev,
4719                struct rte_flow *flow,
4720                const struct rte_flow_action *actions,
4721                void *data,
4722                struct rte_flow_error *error)
4723 {
4724         const struct mlx5_flow_driver_ops *fops;
4725         enum mlx5_flow_drv_type ftype = flow->drv_type;
4726
4727         assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
4728         fops = flow_get_drv_ops(ftype);
4729
4730         return fops->query(dev, flow, actions, data, error);
4731 }
4732
4733 /**
4734  * Query a flow.
4735  *
4736  * @see rte_flow_query()
4737  * @see rte_flow_ops
4738  */
4739 int
4740 mlx5_flow_query(struct rte_eth_dev *dev,
4741                 struct rte_flow *flow,
4742                 const struct rte_flow_action *actions,
4743                 void *data,
4744                 struct rte_flow_error *error)
4745 {
4746         int ret;
4747
4748         ret = flow_drv_query(dev, flow, actions, data, error);
4749         if (ret < 0)
4750                 return ret;
4751         return 0;
4752 }
4753
4754 /**
4755  * Convert a flow director filter to a generic flow.
4756  *
4757  * @param dev
4758  *   Pointer to Ethernet device.
4759  * @param fdir_filter
4760  *   Flow director filter to add.
4761  * @param attributes
4762  *   Generic flow parameters structure.
4763  *
4764  * @return
4765  *   0 on success, a negative errno value otherwise and rte_errno is set.
4766  */
4767 static int
4768 flow_fdir_filter_convert(struct rte_eth_dev *dev,
4769                          const struct rte_eth_fdir_filter *fdir_filter,
4770                          struct mlx5_fdir *attributes)
4771 {
4772         struct mlx5_priv *priv = dev->data->dev_private;
4773         const struct rte_eth_fdir_input *input = &fdir_filter->input;
4774         const struct rte_eth_fdir_masks *mask =
4775                 &dev->data->dev_conf.fdir_conf.mask;
4776
4777         /* Validate queue number. */
4778         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
4779                 DRV_LOG(ERR, "port %u invalid queue number %d",
4780                         dev->data->port_id, fdir_filter->action.rx_queue);
4781                 rte_errno = EINVAL;
4782                 return -rte_errno;
4783         }
4784         attributes->attr.ingress = 1;
4785         attributes->items[0] = (struct rte_flow_item) {
4786                 .type = RTE_FLOW_ITEM_TYPE_ETH,
4787                 .spec = &attributes->l2,
4788                 .mask = &attributes->l2_mask,
4789         };
4790         switch (fdir_filter->action.behavior) {
4791         case RTE_ETH_FDIR_ACCEPT:
4792                 attributes->actions[0] = (struct rte_flow_action){
4793                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
4794                         .conf = &attributes->queue,
4795                 };
4796                 break;
4797         case RTE_ETH_FDIR_REJECT:
4798                 attributes->actions[0] = (struct rte_flow_action){
4799                         .type = RTE_FLOW_ACTION_TYPE_DROP,
4800                 };
4801                 break;
4802         default:
4803                 DRV_LOG(ERR, "port %u invalid behavior %d",
4804                         dev->data->port_id,
4805                         fdir_filter->action.behavior);
4806                 rte_errno = ENOTSUP;
4807                 return -rte_errno;
4808         }
4809         attributes->queue.index = fdir_filter->action.rx_queue;
4810         /* Handle L3. */
4811         switch (fdir_filter->input.flow_type) {
4812         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
4813         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
4814         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
4815                 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
4816                         .src_addr = input->flow.ip4_flow.src_ip,
4817                         .dst_addr = input->flow.ip4_flow.dst_ip,
4818                         .time_to_live = input->flow.ip4_flow.ttl,
4819                         .type_of_service = input->flow.ip4_flow.tos,
4820                 };
4821                 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
4822                         .src_addr = mask->ipv4_mask.src_ip,
4823                         .dst_addr = mask->ipv4_mask.dst_ip,
4824                         .time_to_live = mask->ipv4_mask.ttl,
4825                         .type_of_service = mask->ipv4_mask.tos,
4826                         .next_proto_id = mask->ipv4_mask.proto,
4827                 };
4828                 attributes->items[1] = (struct rte_flow_item){
4829                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
4830                         .spec = &attributes->l3,
4831                         .mask = &attributes->l3_mask,
4832                 };
4833                 break;
4834         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
4835         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
4836         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
4837                 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
4838                         .hop_limits = input->flow.ipv6_flow.hop_limits,
4839                         .proto = input->flow.ipv6_flow.proto,
4840                 };
4841
4842                 memcpy(attributes->l3.ipv6.hdr.src_addr,
4843                        input->flow.ipv6_flow.src_ip,
4844                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
4845                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
4846                        input->flow.ipv6_flow.dst_ip,
4847                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
4848                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
4849                        mask->ipv6_mask.src_ip,
4850                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
4851                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
4852                        mask->ipv6_mask.dst_ip,
4853                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
4854                 attributes->items[1] = (struct rte_flow_item){
4855                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
4856                         .spec = &attributes->l3,
4857                         .mask = &attributes->l3_mask,
4858                 };
4859                 break;
4860         default:
4861                 DRV_LOG(ERR, "port %u invalid flow type%d",
4862                         dev->data->port_id, fdir_filter->input.flow_type);
4863                 rte_errno = ENOTSUP;
4864                 return -rte_errno;
4865         }
4866         /* Handle L4. */
4867         switch (fdir_filter->input.flow_type) {
4868         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
4869                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
4870                         .src_port = input->flow.udp4_flow.src_port,
4871                         .dst_port = input->flow.udp4_flow.dst_port,
4872                 };
4873                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
4874                         .src_port = mask->src_port_mask,
4875                         .dst_port = mask->dst_port_mask,
4876                 };
4877                 attributes->items[2] = (struct rte_flow_item){
4878                         .type = RTE_FLOW_ITEM_TYPE_UDP,
4879                         .spec = &attributes->l4,
4880                         .mask = &attributes->l4_mask,
4881                 };
4882                 break;
4883         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
4884                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
4885                         .src_port = input->flow.tcp4_flow.src_port,
4886                         .dst_port = input->flow.tcp4_flow.dst_port,
4887                 };
4888                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
4889                         .src_port = mask->src_port_mask,
4890                         .dst_port = mask->dst_port_mask,
4891                 };
4892                 attributes->items[2] = (struct rte_flow_item){
4893                         .type = RTE_FLOW_ITEM_TYPE_TCP,
4894                         .spec = &attributes->l4,
4895                         .mask = &attributes->l4_mask,
4896                 };
4897                 break;
4898         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
4899                 attributes->l4.udp.hdr = (struct rte_udp_hdr){
4900                         .src_port = input->flow.udp6_flow.src_port,
4901                         .dst_port = input->flow.udp6_flow.dst_port,
4902                 };
4903                 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
4904                         .src_port = mask->src_port_mask,
4905                         .dst_port = mask->dst_port_mask,
4906                 };
4907                 attributes->items[2] = (struct rte_flow_item){
4908                         .type = RTE_FLOW_ITEM_TYPE_UDP,
4909                         .spec = &attributes->l4,
4910                         .mask = &attributes->l4_mask,
4911                 };
4912                 break;
4913         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
4914                 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
4915                         .src_port = input->flow.tcp6_flow.src_port,
4916                         .dst_port = input->flow.tcp6_flow.dst_port,
4917                 };
4918                 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
4919                         .src_port = mask->src_port_mask,
4920                         .dst_port = mask->dst_port_mask,
4921                 };
4922                 attributes->items[2] = (struct rte_flow_item){
4923                         .type = RTE_FLOW_ITEM_TYPE_TCP,
4924                         .spec = &attributes->l4,
4925                         .mask = &attributes->l4_mask,
4926                 };
4927                 break;
4928         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
4929         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
4930                 break;
4931         default:
4932                 DRV_LOG(ERR, "port %u invalid flow type%d",
4933                         dev->data->port_id, fdir_filter->input.flow_type);
4934                 rte_errno = ENOTSUP;
4935                 return -rte_errno;
4936         }
4937         return 0;
4938 }
4939
4940 #define FLOW_FDIR_CMP(f1, f2, fld) \
4941         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
4942
4943 /**
4944  * Compare two FDIR flows. If items and actions are identical, the two flows are
4945  * regarded as same.
4946  *
4947  * @param dev
4948  *   Pointer to Ethernet device.
4949  * @param f1
4950  *   FDIR flow to compare.
4951  * @param f2
4952  *   FDIR flow to compare.
4953  *
4954  * @return
4955  *   Zero on match, 1 otherwise.
4956  */
4957 static int
4958 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
4959 {
4960         if (FLOW_FDIR_CMP(f1, f2, attr) ||
4961             FLOW_FDIR_CMP(f1, f2, l2) ||
4962             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
4963             FLOW_FDIR_CMP(f1, f2, l3) ||
4964             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
4965             FLOW_FDIR_CMP(f1, f2, l4) ||
4966             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
4967             FLOW_FDIR_CMP(f1, f2, actions[0].type))
4968                 return 1;
4969         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
4970             FLOW_FDIR_CMP(f1, f2, queue))
4971                 return 1;
4972         return 0;
4973 }
4974
4975 /**
4976  * Search device flow list to find out a matched FDIR flow.
4977  *
4978  * @param dev
4979  *   Pointer to Ethernet device.
4980  * @param fdir_flow
4981  *   FDIR flow to lookup.
4982  *
4983  * @return
4984  *   Pointer of flow if found, NULL otherwise.
4985  */
4986 static struct rte_flow *
4987 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
4988 {
4989         struct mlx5_priv *priv = dev->data->dev_private;
4990         struct rte_flow *flow = NULL;
4991
4992         assert(fdir_flow);
4993         TAILQ_FOREACH(flow, &priv->flows, next) {
4994                 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) {
4995                         DRV_LOG(DEBUG, "port %u found FDIR flow %p",
4996                                 dev->data->port_id, (void *)flow);
4997                         break;
4998                 }
4999         }
5000         return flow;
5001 }
5002
5003 /**
5004  * Add new flow director filter and store it in list.
5005  *
5006  * @param dev
5007  *   Pointer to Ethernet device.
5008  * @param fdir_filter
5009  *   Flow director filter to add.
5010  *
5011  * @return
5012  *   0 on success, a negative errno value otherwise and rte_errno is set.
5013  */
5014 static int
5015 flow_fdir_filter_add(struct rte_eth_dev *dev,
5016                      const struct rte_eth_fdir_filter *fdir_filter)
5017 {
5018         struct mlx5_priv *priv = dev->data->dev_private;
5019         struct mlx5_fdir *fdir_flow;
5020         struct rte_flow *flow;
5021         int ret;
5022
5023         fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0);
5024         if (!fdir_flow) {
5025                 rte_errno = ENOMEM;
5026                 return -rte_errno;
5027         }
5028         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
5029         if (ret)
5030                 goto error;
5031         flow = flow_fdir_filter_lookup(dev, fdir_flow);
5032         if (flow) {
5033                 rte_errno = EEXIST;
5034                 goto error;
5035         }
5036         flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
5037                                 fdir_flow->items, fdir_flow->actions, true,
5038                                 NULL);
5039         if (!flow)
5040                 goto error;
5041         assert(!flow->fdir);
5042         flow->fdir = fdir_flow;
5043         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
5044                 dev->data->port_id, (void *)flow);
5045         return 0;
5046 error:
5047         rte_free(fdir_flow);
5048         return -rte_errno;
5049 }
5050
5051 /**
5052  * Delete specific filter.
5053  *
5054  * @param dev
5055  *   Pointer to Ethernet device.
5056  * @param fdir_filter
5057  *   Filter to be deleted.
5058  *
5059  * @return
5060  *   0 on success, a negative errno value otherwise and rte_errno is set.
5061  */
5062 static int
5063 flow_fdir_filter_delete(struct rte_eth_dev *dev,
5064                         const struct rte_eth_fdir_filter *fdir_filter)
5065 {
5066         struct mlx5_priv *priv = dev->data->dev_private;
5067         struct rte_flow *flow;
5068         struct mlx5_fdir fdir_flow = {
5069                 .attr.group = 0,
5070         };
5071         int ret;
5072
5073         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
5074         if (ret)
5075                 return -rte_errno;
5076         flow = flow_fdir_filter_lookup(dev, &fdir_flow);
5077         if (!flow) {
5078                 rte_errno = ENOENT;
5079                 return -rte_errno;
5080         }
5081         flow_list_destroy(dev, &priv->flows, flow);
5082         DRV_LOG(DEBUG, "port %u deleted FDIR flow %p",
5083                 dev->data->port_id, (void *)flow);
5084         return 0;
5085 }
5086
5087 /**
5088  * Update queue for specific filter.
5089  *
5090  * @param dev
5091  *   Pointer to Ethernet device.
5092  * @param fdir_filter
5093  *   Filter to be updated.
5094  *
5095  * @return
5096  *   0 on success, a negative errno value otherwise and rte_errno is set.
5097  */
5098 static int
5099 flow_fdir_filter_update(struct rte_eth_dev *dev,
5100                         const struct rte_eth_fdir_filter *fdir_filter)
5101 {
5102         int ret;
5103
5104         ret = flow_fdir_filter_delete(dev, fdir_filter);
5105         if (ret)
5106                 return ret;
5107         return flow_fdir_filter_add(dev, fdir_filter);
5108 }
5109
5110 /**
5111  * Flush all filters.
5112  *
5113  * @param dev
5114  *   Pointer to Ethernet device.
5115  */
5116 static void
5117 flow_fdir_filter_flush(struct rte_eth_dev *dev)
5118 {
5119         struct mlx5_priv *priv = dev->data->dev_private;
5120
5121         mlx5_flow_list_flush(dev, &priv->flows);
5122 }
5123
5124 /**
5125  * Get flow director information.
5126  *
5127  * @param dev
5128  *   Pointer to Ethernet device.
5129  * @param[out] fdir_info
5130  *   Resulting flow director information.
5131  */
5132 static void
5133 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
5134 {
5135         struct rte_eth_fdir_masks *mask =
5136                 &dev->data->dev_conf.fdir_conf.mask;
5137
5138         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
5139         fdir_info->guarant_spc = 0;
5140         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
5141         fdir_info->max_flexpayload = 0;
5142         fdir_info->flow_types_mask[0] = 0;
5143         fdir_info->flex_payload_unit = 0;
5144         fdir_info->max_flex_payload_segment_num = 0;
5145         fdir_info->flex_payload_limit = 0;
5146         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
5147 }
5148
5149 /**
5150  * Deal with flow director operations.
5151  *
5152  * @param dev
5153  *   Pointer to Ethernet device.
5154  * @param filter_op
5155  *   Operation to perform.
5156  * @param arg
5157  *   Pointer to operation-specific structure.
5158  *
5159  * @return
5160  *   0 on success, a negative errno value otherwise and rte_errno is set.
5161  */
5162 static int
5163 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
5164                     void *arg)
5165 {
5166         enum rte_fdir_mode fdir_mode =
5167                 dev->data->dev_conf.fdir_conf.mode;
5168
5169         if (filter_op == RTE_ETH_FILTER_NOP)
5170                 return 0;
5171         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
5172             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
5173                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
5174                         dev->data->port_id, fdir_mode);
5175                 rte_errno = EINVAL;
5176                 return -rte_errno;
5177         }
5178         switch (filter_op) {
5179         case RTE_ETH_FILTER_ADD:
5180                 return flow_fdir_filter_add(dev, arg);
5181         case RTE_ETH_FILTER_UPDATE:
5182                 return flow_fdir_filter_update(dev, arg);
5183         case RTE_ETH_FILTER_DELETE:
5184                 return flow_fdir_filter_delete(dev, arg);
5185         case RTE_ETH_FILTER_FLUSH:
5186                 flow_fdir_filter_flush(dev);
5187                 break;
5188         case RTE_ETH_FILTER_INFO:
5189                 flow_fdir_info_get(dev, arg);
5190                 break;
5191         default:
5192                 DRV_LOG(DEBUG, "port %u unknown operation %u",
5193                         dev->data->port_id, filter_op);
5194                 rte_errno = EINVAL;
5195                 return -rte_errno;
5196         }
5197         return 0;
5198 }
5199
5200 /**
5201  * Manage filter operations.
5202  *
5203  * @param dev
5204  *   Pointer to Ethernet device structure.
5205  * @param filter_type
5206  *   Filter type.
5207  * @param filter_op
5208  *   Operation to perform.
5209  * @param arg
5210  *   Pointer to operation-specific structure.
5211  *
5212  * @return
5213  *   0 on success, a negative errno value otherwise and rte_errno is set.
5214  */
5215 int
5216 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
5217                      enum rte_filter_type filter_type,
5218                      enum rte_filter_op filter_op,
5219                      void *arg)
5220 {
5221         switch (filter_type) {
5222         case RTE_ETH_FILTER_GENERIC:
5223                 if (filter_op != RTE_ETH_FILTER_GET) {
5224                         rte_errno = EINVAL;
5225                         return -rte_errno;
5226                 }
5227                 *(const void **)arg = &mlx5_flow_ops;
5228                 return 0;
5229         case RTE_ETH_FILTER_FDIR:
5230                 return flow_fdir_ctrl_func(dev, filter_op, arg);
5231         default:
5232                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
5233                         dev->data->port_id, filter_type);
5234                 rte_errno = ENOTSUP;
5235                 return -rte_errno;
5236         }
5237         return 0;
5238 }
5239
5240 /**
5241  * Create the needed meter and suffix tables.
5242  *
5243  * @param[in] dev
5244  *   Pointer to Ethernet device.
5245  * @param[in] fm
5246  *   Pointer to the flow meter.
5247  *
5248  * @return
5249  *   Pointer to table set on success, NULL otherwise.
5250  */
5251 struct mlx5_meter_domains_infos *
5252 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
5253                           const struct mlx5_flow_meter *fm)
5254 {
5255         const struct mlx5_flow_driver_ops *fops;
5256
5257         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5258         return fops->create_mtr_tbls(dev, fm);
5259 }
5260
5261 /**
5262  * Destroy the meter table set.
5263  *
5264  * @param[in] dev
5265  *   Pointer to Ethernet device.
5266  * @param[in] tbl
5267  *   Pointer to the meter table set.
5268  *
5269  * @return
5270  *   0 on success.
5271  */
5272 int
5273 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
5274                            struct mlx5_meter_domains_infos *tbls)
5275 {
5276         const struct mlx5_flow_driver_ops *fops;
5277
5278         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5279         return fops->destroy_mtr_tbls(dev, tbls);
5280 }
5281
5282 /**
5283  * Create policer rules.
5284  *
5285  * @param[in] dev
5286  *   Pointer to Ethernet device.
5287  * @param[in] fm
5288  *   Pointer to flow meter structure.
5289  * @param[in] attr
5290  *   Pointer to flow attributes.
5291  *
5292  * @return
5293  *   0 on success, -1 otherwise.
5294  */
5295 int
5296 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
5297                                struct mlx5_flow_meter *fm,
5298                                const struct rte_flow_attr *attr)
5299 {
5300         const struct mlx5_flow_driver_ops *fops;
5301
5302         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5303         return fops->create_policer_rules(dev, fm, attr);
5304 }
5305
5306 /**
5307  * Destroy policer rules.
5308  *
5309  * @param[in] fm
5310  *   Pointer to flow meter structure.
5311  * @param[in] attr
5312  *   Pointer to flow attributes.
5313  *
5314  * @return
5315  *   0 on success, -1 otherwise.
5316  */
5317 int
5318 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
5319                                 struct mlx5_flow_meter *fm,
5320                                 const struct rte_flow_attr *attr)
5321 {
5322         const struct mlx5_flow_driver_ops *fops;
5323
5324         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5325         return fops->destroy_policer_rules(dev, fm, attr);
5326 }
5327
5328 /**
5329  * Allocate a counter.
5330  *
5331  * @param[in] dev
5332  *   Pointer to Ethernet device structure.
5333  *
5334  * @return
5335  *   Pointer to allocated counter  on success, NULL otherwise.
5336  */
5337 struct mlx5_flow_counter *
5338 mlx5_counter_alloc(struct rte_eth_dev *dev)
5339 {
5340         const struct mlx5_flow_driver_ops *fops;
5341         struct rte_flow_attr attr = { .transfer = 0 };
5342
5343         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5344                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5345                 return fops->counter_alloc(dev);
5346         }
5347         DRV_LOG(ERR,
5348                 "port %u counter allocate is not supported.",
5349                  dev->data->port_id);
5350         return NULL;
5351 }
5352
5353 /**
5354  * Free a counter.
5355  *
5356  * @param[in] dev
5357  *   Pointer to Ethernet device structure.
5358  * @param[in] cnt
5359  *   Pointer to counter to be free.
5360  */
5361 void
5362 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt)
5363 {
5364         const struct mlx5_flow_driver_ops *fops;
5365         struct rte_flow_attr attr = { .transfer = 0 };
5366
5367         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5368                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5369                 fops->counter_free(dev, cnt);
5370                 return;
5371         }
5372         DRV_LOG(ERR,
5373                 "port %u counter free is not supported.",
5374                  dev->data->port_id);
5375 }
5376
5377 /**
5378  * Query counter statistics.
5379  *
5380  * @param[in] dev
5381  *   Pointer to Ethernet device structure.
5382  * @param[in] cnt
5383  *   Pointer to counter to query.
5384  * @param[in] clear
5385  *   Set to clear counter statistics.
5386  * @param[out] pkts
5387  *   The counter hits packets number to save.
5388  * @param[out] bytes
5389  *   The counter hits bytes number to save.
5390  *
5391  * @return
5392  *   0 on success, a negative errno value otherwise.
5393  */
5394 int
5395 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt,
5396                    bool clear, uint64_t *pkts, uint64_t *bytes)
5397 {
5398         const struct mlx5_flow_driver_ops *fops;
5399         struct rte_flow_attr attr = { .transfer = 0 };
5400
5401         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5402                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5403                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
5404         }
5405         DRV_LOG(ERR,
5406                 "port %u counter query is not supported.",
5407                  dev->data->port_id);
5408         return -ENOTSUP;
5409 }
5410
5411 #define MLX5_POOL_QUERY_FREQ_US 1000000
5412
5413 /**
5414  * Set the periodic procedure for triggering asynchronous batch queries for all
5415  * the counter pools.
5416  *
5417  * @param[in] sh
5418  *   Pointer to mlx5_ibv_shared object.
5419  */
5420 void
5421 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh)
5422 {
5423         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0);
5424         uint32_t pools_n = rte_atomic16_read(&cont->n_valid);
5425         uint32_t us;
5426
5427         cont = MLX5_CNT_CONTAINER(sh, 1, 0);
5428         pools_n += rte_atomic16_read(&cont->n_valid);
5429         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
5430         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
5431         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
5432                 sh->cmng.query_thread_on = 0;
5433                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
5434         } else {
5435                 sh->cmng.query_thread_on = 1;
5436         }
5437 }
5438
5439 /**
5440  * The periodic procedure for triggering asynchronous batch queries for all the
5441  * counter pools. This function is probably called by the host thread.
5442  *
5443  * @param[in] arg
5444  *   The parameter for the alarm process.
5445  */
5446 void
5447 mlx5_flow_query_alarm(void *arg)
5448 {
5449         struct mlx5_ibv_shared *sh = arg;
5450         struct mlx5_devx_obj *dcs;
5451         uint16_t offset;
5452         int ret;
5453         uint8_t batch = sh->cmng.batch;
5454         uint16_t pool_index = sh->cmng.pool_index;
5455         struct mlx5_pools_container *cont;
5456         struct mlx5_pools_container *mcont;
5457         struct mlx5_flow_counter_pool *pool;
5458
5459         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
5460                 goto set_alarm;
5461 next_container:
5462         cont = MLX5_CNT_CONTAINER(sh, batch, 1);
5463         mcont = MLX5_CNT_CONTAINER(sh, batch, 0);
5464         /* Check if resize was done and need to flip a container. */
5465         if (cont != mcont) {
5466                 if (cont->pools) {
5467                         /* Clean the old container. */
5468                         rte_free(cont->pools);
5469                         memset(cont, 0, sizeof(*cont));
5470                 }
5471                 rte_cio_wmb();
5472                  /* Flip the host container. */
5473                 sh->cmng.mhi[batch] ^= (uint8_t)2;
5474                 cont = mcont;
5475         }
5476         if (!cont->pools) {
5477                 /* 2 empty containers case is unexpected. */
5478                 if (unlikely(batch != sh->cmng.batch))
5479                         goto set_alarm;
5480                 batch ^= 0x1;
5481                 pool_index = 0;
5482                 goto next_container;
5483         }
5484         pool = cont->pools[pool_index];
5485         if (pool->raw_hw)
5486                 /* There is a pool query in progress. */
5487                 goto set_alarm;
5488         pool->raw_hw =
5489                 LIST_FIRST(&sh->cmng.free_stat_raws);
5490         if (!pool->raw_hw)
5491                 /* No free counter statistics raw memory. */
5492                 goto set_alarm;
5493         dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
5494                                                               (&pool->a64_dcs);
5495         offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
5496         ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
5497                                                offset, NULL, NULL,
5498                                                pool->raw_hw->mem_mng->dm->id,
5499                                                (void *)(uintptr_t)
5500                                                (pool->raw_hw->data + offset),
5501                                                sh->devx_comp,
5502                                                (uint64_t)(uintptr_t)pool);
5503         if (ret) {
5504                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
5505                         " %d", pool->min_dcs->id);
5506                 pool->raw_hw = NULL;
5507                 goto set_alarm;
5508         }
5509         pool->raw_hw->min_dcs_id = dcs->id;
5510         LIST_REMOVE(pool->raw_hw, next);
5511         sh->cmng.pending_queries++;
5512         pool_index++;
5513         if (pool_index >= rte_atomic16_read(&cont->n_valid)) {
5514                 batch ^= 0x1;
5515                 pool_index = 0;
5516         }
5517 set_alarm:
5518         sh->cmng.batch = batch;
5519         sh->cmng.pool_index = pool_index;
5520         mlx5_set_query_alarm(sh);
5521 }
5522
5523 /**
5524  * Handler for the HW respond about ready values from an asynchronous batch
5525  * query. This function is probably called by the host thread.
5526  *
5527  * @param[in] sh
5528  *   The pointer to the shared IB device context.
5529  * @param[in] async_id
5530  *   The Devx async ID.
5531  * @param[in] status
5532  *   The status of the completion.
5533  */
5534 void
5535 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh,
5536                                   uint64_t async_id, int status)
5537 {
5538         struct mlx5_flow_counter_pool *pool =
5539                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
5540         struct mlx5_counter_stats_raw *raw_to_free;
5541
5542         if (unlikely(status)) {
5543                 raw_to_free = pool->raw_hw;
5544         } else {
5545                 raw_to_free = pool->raw;
5546                 rte_spinlock_lock(&pool->sl);
5547                 pool->raw = pool->raw_hw;
5548                 rte_spinlock_unlock(&pool->sl);
5549                 rte_atomic64_add(&pool->query_gen, 1);
5550                 /* Be sure the new raw counters data is updated in memory. */
5551                 rte_cio_wmb();
5552         }
5553         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
5554         pool->raw_hw = NULL;
5555         sh->cmng.pending_queries--;
5556 }
5557
5558 /**
5559  * Translate the rte_flow group index to HW table value.
5560  *
5561  * @param[in] attributes
5562  *   Pointer to flow attributes
5563  * @param[in] external
5564  *   Value is part of flow rule created by request external to PMD.
5565  * @param[in] group
5566  *   rte_flow group index value.
5567  * @param[out] table
5568  *   HW table value.
5569  * @param[out] error
5570  *   Pointer to error structure.
5571  *
5572  * @return
5573  *   0 on success, a negative errno value otherwise and rte_errno is set.
5574  */
5575 int
5576 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external,
5577                          uint32_t group, uint32_t *table,
5578                          struct rte_flow_error *error)
5579 {
5580         if (attributes->transfer && external) {
5581                 if (group == UINT32_MAX)
5582                         return rte_flow_error_set
5583                                                 (error, EINVAL,
5584                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5585                                                  NULL,
5586                                                  "group index not supported");
5587                 *table = group + 1;
5588         } else {
5589                 *table = group;
5590         }
5591         return 0;
5592 }
5593
5594 /**
5595  * Discover availability of metadata reg_c's.
5596  *
5597  * Iteratively use test flows to check availability.
5598  *
5599  * @param[in] dev
5600  *   Pointer to the Ethernet device structure.
5601  *
5602  * @return
5603  *   0 on success, a negative errno value otherwise and rte_errno is set.
5604  */
5605 int
5606 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
5607 {
5608         struct mlx5_priv *priv = dev->data->dev_private;
5609         struct mlx5_dev_config *config = &priv->config;
5610         enum modify_reg idx;
5611         int n = 0;
5612
5613         /* reg_c[0] and reg_c[1] are reserved. */
5614         config->flow_mreg_c[n++] = REG_C_0;
5615         config->flow_mreg_c[n++] = REG_C_1;
5616         /* Discover availability of other reg_c's. */
5617         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
5618                 struct rte_flow_attr attr = {
5619                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5620                         .priority = MLX5_FLOW_PRIO_RSVD,
5621                         .ingress = 1,
5622                 };
5623                 struct rte_flow_item items[] = {
5624                         [0] = {
5625                                 .type = RTE_FLOW_ITEM_TYPE_END,
5626                         },
5627                 };
5628                 struct rte_flow_action actions[] = {
5629                         [0] = {
5630                                 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5631                                 .conf = &(struct mlx5_flow_action_copy_mreg){
5632                                         .src = REG_C_1,
5633                                         .dst = idx,
5634                                 },
5635                         },
5636                         [1] = {
5637                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5638                                 .conf = &(struct rte_flow_action_jump){
5639                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5640                                 },
5641                         },
5642                         [2] = {
5643                                 .type = RTE_FLOW_ACTION_TYPE_END,
5644                         },
5645                 };
5646                 struct rte_flow *flow;
5647                 struct rte_flow_error error;
5648
5649                 if (!config->dv_flow_en)
5650                         break;
5651                 /* Create internal flow, validation skips copy action. */
5652                 flow = flow_list_create(dev, NULL, &attr, items,
5653                                         actions, false, &error);
5654                 if (!flow)
5655                         continue;
5656                 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
5657                         config->flow_mreg_c[n++] = idx;
5658                 flow_list_destroy(dev, NULL, flow);
5659         }
5660         for (; n < MLX5_MREG_C_NUM; ++n)
5661                 config->flow_mreg_c[n] = REG_NONE;
5662         return 0;
5663 }
5664
5665 /**
5666  * Dump flow raw hw data to file
5667  *
5668  * @param[in] dev
5669  *    The pointer to Ethernet device.
5670  * @param[in] file
5671  *   A pointer to a file for output.
5672  * @param[out] error
5673  *   Perform verbose error reporting if not NULL. PMDs initialize this
5674  *   structure in case of error only.
5675  * @return
5676  *   0 on success, a nagative value otherwise.
5677  */
5678 int
5679 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
5680                    FILE *file,
5681                    struct rte_flow_error *error __rte_unused)
5682 {
5683         struct mlx5_priv *priv = dev->data->dev_private;
5684
5685         return mlx5_devx_cmd_flow_dump(priv->sh, file);
5686 }