drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11
  12 /* Verbs header. */
  13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  14 #ifdef PEDANTIC
  15 #pragma GCC diagnostic ignored "-Wpedantic"
  16 #endif
  17 #include <infiniband/verbs.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_common.h>
  23 #include <rte_ether.h>
  24 #include <rte_eth_ctrl.h>
  25 #include <rte_ethdev_driver.h>
  26 #include <rte_flow.h>
  27 #include <rte_flow_driver.h>
  28 #include <rte_malloc.h>
  29 #include <rte_ip.h>
  30
  31 #include "mlx5.h"
  32 #include "mlx5_defs.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_glue.h"
  35 #include "mlx5_prm.h"
  36 #include "mlx5_rxtx.h"
  37
  38 /* Dev ops structure defined in mlx5.c */
  39 extern const struct eth_dev_ops mlx5_dev_ops;
  40 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
  41
  42 /** Device flow drivers. */
  43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  44 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
  45 #endif
  46 extern const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops;
  47 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  48
  49 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  50
  51 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  52         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  53 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  54         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  55 #endif
  56         [MLX5_FLOW_TYPE_TCF] = &mlx5_flow_tcf_drv_ops,
  57         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  58         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  59 };
  60
  61 enum mlx5_expansion {
  62         MLX5_EXPANSION_ROOT,
  63         MLX5_EXPANSION_ROOT_OUTER,
  64         MLX5_EXPANSION_ROOT_ETH_VLAN,
  65         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
  66         MLX5_EXPANSION_OUTER_ETH,
  67         MLX5_EXPANSION_OUTER_ETH_VLAN,
  68         MLX5_EXPANSION_OUTER_VLAN,
  69         MLX5_EXPANSION_OUTER_IPV4,
  70         MLX5_EXPANSION_OUTER_IPV4_UDP,
  71         MLX5_EXPANSION_OUTER_IPV4_TCP,
  72         MLX5_EXPANSION_OUTER_IPV6,
  73         MLX5_EXPANSION_OUTER_IPV6_UDP,
  74         MLX5_EXPANSION_OUTER_IPV6_TCP,
  75         MLX5_EXPANSION_VXLAN,
  76         MLX5_EXPANSION_VXLAN_GPE,
  77         MLX5_EXPANSION_GRE,
  78         MLX5_EXPANSION_MPLS,
  79         MLX5_EXPANSION_ETH,
  80         MLX5_EXPANSION_ETH_VLAN,
  81         MLX5_EXPANSION_VLAN,
  82         MLX5_EXPANSION_IPV4,
  83         MLX5_EXPANSION_IPV4_UDP,
  84         MLX5_EXPANSION_IPV4_TCP,
  85         MLX5_EXPANSION_IPV6,
  86         MLX5_EXPANSION_IPV6_UDP,
  87         MLX5_EXPANSION_IPV6_TCP,
  88 };
  89
  90 /** Supported expansion of items. */
  91 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
  92         [MLX5_EXPANSION_ROOT] = {
  93                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
  94                                                  MLX5_EXPANSION_IPV4,
  95                                                  MLX5_EXPANSION_IPV6),
  96                 .type = RTE_FLOW_ITEM_TYPE_END,
  97         },
  98         [MLX5_EXPANSION_ROOT_OUTER] = {
  99                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 100                                                  MLX5_EXPANSION_OUTER_IPV4,
 101                                                  MLX5_EXPANSION_OUTER_IPV6),
 102                 .type = RTE_FLOW_ITEM_TYPE_END,
 103         },
 104         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 105                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 106                 .type = RTE_FLOW_ITEM_TYPE_END,
 107         },
 108         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 109                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
 110                 .type = RTE_FLOW_ITEM_TYPE_END,
 111         },
 112         [MLX5_EXPANSION_OUTER_ETH] = {
 113                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 114                                                  MLX5_EXPANSION_OUTER_IPV6,
 115                                                  MLX5_EXPANSION_MPLS),
 116                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 117                 .rss_types = 0,
 118         },
 119         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 120                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 121                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 122                 .rss_types = 0,
 123         },
 124         [MLX5_EXPANSION_OUTER_VLAN] = {
 125                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 126                                                  MLX5_EXPANSION_OUTER_IPV6),
 127                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 128         },
 129         [MLX5_EXPANSION_OUTER_IPV4] = {
 130                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 131                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 132                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 133                          MLX5_EXPANSION_GRE),
 134                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 135                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 136                         ETH_RSS_NONFRAG_IPV4_OTHER,
 137         },
 138         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 139                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 140                                                  MLX5_EXPANSION_VXLAN_GPE),
 141                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 142                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 143         },
 144         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 145                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 146                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 147         },
 148         [MLX5_EXPANSION_OUTER_IPV6] = {
 149                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 150                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 151                          MLX5_EXPANSION_OUTER_IPV6_TCP),
 152                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 153                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 154                         ETH_RSS_NONFRAG_IPV6_OTHER,
 155         },
 156         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 157                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 158                                                  MLX5_EXPANSION_VXLAN_GPE),
 159                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 160                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 161         },
 162         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 163                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 164                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 165         },
 166         [MLX5_EXPANSION_VXLAN] = {
 167                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 168                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 169         },
 170         [MLX5_EXPANSION_VXLAN_GPE] = {
 171                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 172                                                  MLX5_EXPANSION_IPV4,
 173                                                  MLX5_EXPANSION_IPV6),
 174                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 175         },
 176         [MLX5_EXPANSION_GRE] = {
 177                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 178                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 179         },
 180         [MLX5_EXPANSION_MPLS] = {
 181                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 182                                                  MLX5_EXPANSION_IPV6),
 183                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 184         },
 185         [MLX5_EXPANSION_ETH] = {
 186                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 187                                                  MLX5_EXPANSION_IPV6),
 188                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 189         },
 190         [MLX5_EXPANSION_ETH_VLAN] = {
 191                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 192                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 193         },
 194         [MLX5_EXPANSION_VLAN] = {
 195                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 196                                                  MLX5_EXPANSION_IPV6),
 197                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 198         },
 199         [MLX5_EXPANSION_IPV4] = {
 200                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 201                                                  MLX5_EXPANSION_IPV4_TCP),
 202                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 203                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 204                         ETH_RSS_NONFRAG_IPV4_OTHER,
 205         },
 206         [MLX5_EXPANSION_IPV4_UDP] = {
 207                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 208                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 209         },
 210         [MLX5_EXPANSION_IPV4_TCP] = {
 211                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 212                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 213         },
 214         [MLX5_EXPANSION_IPV6] = {
 215                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 216                                                  MLX5_EXPANSION_IPV6_TCP),
 217                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 218                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 219                         ETH_RSS_NONFRAG_IPV6_OTHER,
 220         },
 221         [MLX5_EXPANSION_IPV6_UDP] = {
 222                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 223                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 224         },
 225         [MLX5_EXPANSION_IPV6_TCP] = {
 226                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 227                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 228         },
 229 };
 230
 231 static const struct rte_flow_ops mlx5_flow_ops = {
 232         .validate = mlx5_flow_validate,
 233         .create = mlx5_flow_create,
 234         .destroy = mlx5_flow_destroy,
 235         .flush = mlx5_flow_flush,
 236         .isolate = mlx5_flow_isolate,
 237         .query = mlx5_flow_query,
 238 };
 239
 240 /* Convert FDIR request to Generic flow. */
 241 struct mlx5_fdir {
 242         struct rte_flow_attr attr;
 243         struct rte_flow_item items[4];
 244         struct rte_flow_item_eth l2;
 245         struct rte_flow_item_eth l2_mask;
 246         union {
 247                 struct rte_flow_item_ipv4 ipv4;
 248                 struct rte_flow_item_ipv6 ipv6;
 249         } l3;
 250         union {
 251                 struct rte_flow_item_ipv4 ipv4;
 252                 struct rte_flow_item_ipv6 ipv6;
 253         } l3_mask;
 254         union {
 255                 struct rte_flow_item_udp udp;
 256                 struct rte_flow_item_tcp tcp;
 257         } l4;
 258         union {
 259                 struct rte_flow_item_udp udp;
 260                 struct rte_flow_item_tcp tcp;
 261         } l4_mask;
 262         struct rte_flow_action actions[2];
 263         struct rte_flow_action_queue queue;
 264 };
 265
 266 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
 267 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
 268         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
 269 };
 270
 271 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
 272 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 273         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
 274         { 9, 10, 11 }, { 12, 13, 14 },
 275 };
 276
 277 /* Tunnel information. */
 278 struct mlx5_flow_tunnel_info {
 279         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 280         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 281 };
 282
 283 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 284         {
 285                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 286                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 287         },
 288         {
 289                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 290                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 291         },
 292         {
 293                 .tunnel = MLX5_FLOW_LAYER_GRE,
 294                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 295         },
 296         {
 297                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 298                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 299         },
 300         {
 301                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 302                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 303         },
 304 };
 305
 306 /**
 307  * Discover the maximum number of priority available.
 308  *
 309  * @param[in] dev
 310  *   Pointer to the Ethernet device structure.
 311  *
 312  * @return
 313  *   number of supported flow priority on success, a negative errno
 314  *   value otherwise and rte_errno is set.
 315  */
 316 int
 317 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 318 {
 319         struct mlx5_priv *priv = dev->data->dev_private;
 320         struct {
 321                 struct ibv_flow_attr attr;
 322                 struct ibv_flow_spec_eth eth;
 323                 struct ibv_flow_spec_action_drop drop;
 324         } flow_attr = {
 325                 .attr = {
 326                         .num_of_specs = 2,
 327                         .port = (uint8_t)priv->ibv_port,
 328                 },
 329                 .eth = {
 330                         .type = IBV_FLOW_SPEC_ETH,
 331                         .size = sizeof(struct ibv_flow_spec_eth),
 332                 },
 333                 .drop = {
 334                         .size = sizeof(struct ibv_flow_spec_action_drop),
 335                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 336                 },
 337         };
 338         struct ibv_flow *flow;
 339         struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 340         uint16_t vprio[] = { 8, 16 };
 341         int i;
 342         int priority = 0;
 343
 344         if (!drop) {
 345                 rte_errno = ENOTSUP;
 346                 return -rte_errno;
 347         }
 348         for (i = 0; i != RTE_DIM(vprio); i++) {
 349                 flow_attr.attr.priority = vprio[i] - 1;
 350                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 351                 if (!flow)
 352                         break;
 353                 claim_zero(mlx5_glue->destroy_flow(flow));
 354                 priority = vprio[i];
 355         }
 356         mlx5_hrxq_drop_release(dev);
 357         switch (priority) {
 358         case 8:
 359                 priority = RTE_DIM(priority_map_3);
 360                 break;
 361         case 16:
 362                 priority = RTE_DIM(priority_map_5);
 363                 break;
 364         default:
 365                 rte_errno = ENOTSUP;
 366                 DRV_LOG(ERR,
 367                         "port %u verbs maximum priority: %d expected 8/16",
 368                         dev->data->port_id, priority);
 369                 return -rte_errno;
 370         }
 371         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 372                 dev->data->port_id, priority);
 373         return priority;
 374 }
 375
 376 /**
 377  * Adjust flow priority based on the highest layer and the request priority.
 378  *
 379  * @param[in] dev
 380  *   Pointer to the Ethernet device structure.
 381  * @param[in] priority
 382  *   The rule base priority.
 383  * @param[in] subpriority
 384  *   The priority based on the items.
 385  *
 386  * @return
 387  *   The new priority.
 388  */
 389 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 390                                    uint32_t subpriority)
 391 {
 392         uint32_t res = 0;
 393         struct mlx5_priv *priv = dev->data->dev_private;
 394
 395         switch (priv->config.flow_prio) {
 396         case RTE_DIM(priority_map_3):
 397                 res = priority_map_3[priority][subpriority];
 398                 break;
 399         case RTE_DIM(priority_map_5):
 400                 res = priority_map_5[priority][subpriority];
 401                 break;
 402         }
 403         return  res;
 404 }
 405
 406 /**
 407  * Verify the @p item specifications (spec, last, mask) are compatible with the
 408  * NIC capabilities.
 409  *
 410  * @param[in] item
 411  *   Item specification.
 412  * @param[in] mask
 413  *   @p item->mask or flow default bit-masks.
 414  * @param[in] nic_mask
 415  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 416  * @param[in] size
 417  *   Bit-masks size in bytes.
 418  * @param[out] error
 419  *   Pointer to error structure.
 420  *
 421  * @return
 422  *   0 on success, a negative errno value otherwise and rte_errno is set.
 423  */
 424 int
 425 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 426                           const uint8_t *mask,
 427                           const uint8_t *nic_mask,
 428                           unsigned int size,
 429                           struct rte_flow_error *error)
 430 {
 431         unsigned int i;
 432
 433         assert(nic_mask);
 434         for (i = 0; i < size; ++i)
 435                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 436                         return rte_flow_error_set(error, ENOTSUP,
 437                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 438                                                   item,
 439                                                   "mask enables non supported"
 440                                                   " bits");
 441         if (!item->spec && (item->mask || item->last))
 442                 return rte_flow_error_set(error, EINVAL,
 443                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 444                                           "mask/last without a spec is not"
 445                                           " supported");
 446         if (item->spec && item->last) {
 447                 uint8_t spec[size];
 448                 uint8_t last[size];
 449                 unsigned int i;
 450                 int ret;
 451
 452                 for (i = 0; i < size; ++i) {
 453                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 454                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 455                 }
 456                 ret = memcmp(spec, last, size);
 457                 if (ret != 0)
 458                         return rte_flow_error_set(error, EINVAL,
 459                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 460                                                   item,
 461                                                   "range is not valid");
 462         }
 463         return 0;
 464 }
 465
 466 /**
 467  * Adjust the hash fields according to the @p flow information.
 468  *
 469  * @param[in] dev_flow.
 470  *   Pointer to the mlx5_flow.
 471  * @param[in] tunnel
 472  *   1 when the hash field is for a tunnel item.
 473  * @param[in] layer_types
 474  *   ETH_RSS_* types.
 475  * @param[in] hash_fields
 476  *   Item hash fields.
 477  *
 478  * @return
 479  *   The hash fileds that should be used.
 480  */
 481 uint64_t
 482 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
 483                             int tunnel __rte_unused, uint64_t layer_types,
 484                             uint64_t hash_fields)
 485 {
 486         struct rte_flow *flow = dev_flow->flow;
 487 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 488         int rss_request_inner = flow->rss.level >= 2;
 489
 490         /* Check RSS hash level for tunnel. */
 491         if (tunnel && rss_request_inner)
 492                 hash_fields |= IBV_RX_HASH_INNER;
 493         else if (tunnel || rss_request_inner)
 494                 return 0;
 495 #endif
 496         /* Check if requested layer matches RSS hash fields. */
 497         if (!(flow->rss.types & layer_types))
 498                 return 0;
 499         return hash_fields;
 500 }
 501
 502 /**
 503  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 504  * if several tunnel rules are used on this queue, the tunnel ptype will be
 505  * cleared.
 506  *
 507  * @param rxq_ctrl
 508  *   Rx queue to update.
 509  */
 510 static void
 511 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 512 {
 513         unsigned int i;
 514         uint32_t tunnel_ptype = 0;
 515
 516         /* Look up for the ptype to use. */
 517         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 518                 if (!rxq_ctrl->flow_tunnels_n[i])
 519                         continue;
 520                 if (!tunnel_ptype) {
 521                         tunnel_ptype = tunnels_info[i].ptype;
 522                 } else {
 523                         tunnel_ptype = 0;
 524                         break;
 525                 }
 526         }
 527         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 528 }
 529
 530 /**
 531  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 532  * flow.
 533  *
 534  * @param[in] dev
 535  *   Pointer to the Ethernet device structure.
 536  * @param[in] dev_flow
 537  *   Pointer to device flow structure.
 538  */
 539 static void
 540 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 541 {
 542         struct mlx5_priv *priv = dev->data->dev_private;
 543         struct rte_flow *flow = dev_flow->flow;
 544         const int mark = !!(flow->actions &
 545                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 546         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 547         unsigned int i;
 548
 549         for (i = 0; i != flow->rss.queue_num; ++i) {
 550                 int idx = (*flow->queue)[i];
 551                 struct mlx5_rxq_ctrl *rxq_ctrl =
 552                         container_of((*priv->rxqs)[idx],
 553                                      struct mlx5_rxq_ctrl, rxq);
 554
 555                 if (mark) {
 556                         rxq_ctrl->rxq.mark = 1;
 557                         rxq_ctrl->flow_mark_n++;
 558                 }
 559                 if (tunnel) {
 560                         unsigned int j;
 561
 562                         /* Increase the counter matching the flow. */
 563                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 564                                 if ((tunnels_info[j].tunnel &
 565                                      dev_flow->layers) ==
 566                                     tunnels_info[j].tunnel) {
 567                                         rxq_ctrl->flow_tunnels_n[j]++;
 568                                         break;
 569                                 }
 570                         }
 571                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 572                 }
 573         }
 574 }
 575
 576 /**
 577  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
 578  *
 579  * @param[in] dev
 580  *   Pointer to the Ethernet device structure.
 581  * @param[in] flow
 582  *   Pointer to flow structure.
 583  */
 584 static void
 585 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 586 {
 587         struct mlx5_flow *dev_flow;
 588
 589         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 590                 flow_drv_rxq_flags_set(dev, dev_flow);
 591 }
 592
 593 /**
 594  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 595  * device flow if no other flow uses it with the same kind of request.
 596  *
 597  * @param dev
 598  *   Pointer to Ethernet device.
 599  * @param[in] dev_flow
 600  *   Pointer to the device flow.
 601  */
 602 static void
 603 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 604 {
 605         struct mlx5_priv *priv = dev->data->dev_private;
 606         struct rte_flow *flow = dev_flow->flow;
 607         const int mark = !!(flow->actions &
 608                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 609         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 610         unsigned int i;
 611
 612         assert(dev->data->dev_started);
 613         for (i = 0; i != flow->rss.queue_num; ++i) {
 614                 int idx = (*flow->queue)[i];
 615                 struct mlx5_rxq_ctrl *rxq_ctrl =
 616                         container_of((*priv->rxqs)[idx],
 617                                      struct mlx5_rxq_ctrl, rxq);
 618
 619                 if (mark) {
 620                         rxq_ctrl->flow_mark_n--;
 621                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 622                 }
 623                 if (tunnel) {
 624                         unsigned int j;
 625
 626                         /* Decrease the counter matching the flow. */
 627                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 628                                 if ((tunnels_info[j].tunnel &
 629                                      dev_flow->layers) ==
 630                                     tunnels_info[j].tunnel) {
 631                                         rxq_ctrl->flow_tunnels_n[j]--;
 632                                         break;
 633                                 }
 634                         }
 635                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 636                 }
 637         }
 638 }
 639
 640 /**
 641  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 642  * @p flow if no other flow uses it with the same kind of request.
 643  *
 644  * @param dev
 645  *   Pointer to Ethernet device.
 646  * @param[in] flow
 647  *   Pointer to the flow.
 648  */
 649 static void
 650 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 651 {
 652         struct mlx5_flow *dev_flow;
 653
 654         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 655                 flow_drv_rxq_flags_trim(dev, dev_flow);
 656 }
 657
 658 /**
 659  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
 660  *
 661  * @param dev
 662  *   Pointer to Ethernet device.
 663  */
 664 static void
 665 flow_rxq_flags_clear(struct rte_eth_dev *dev)
 666 {
 667         struct mlx5_priv *priv = dev->data->dev_private;
 668         unsigned int i;
 669
 670         for (i = 0; i != priv->rxqs_n; ++i) {
 671                 struct mlx5_rxq_ctrl *rxq_ctrl;
 672                 unsigned int j;
 673
 674                 if (!(*priv->rxqs)[i])
 675                         continue;
 676                 rxq_ctrl = container_of((*priv->rxqs)[i],
 677                                         struct mlx5_rxq_ctrl, rxq);
 678                 rxq_ctrl->flow_mark_n = 0;
 679                 rxq_ctrl->rxq.mark = 0;
 680                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
 681                         rxq_ctrl->flow_tunnels_n[j] = 0;
 682                 rxq_ctrl->rxq.tunnel = 0;
 683         }
 684 }
 685
 686 /*
 687  * Validate the flag action.
 688  *
 689  * @param[in] action_flags
 690  *   Bit-fields that holds the actions detected until now.
 691  * @param[in] attr
 692  *   Attributes of flow that includes this action.
 693  * @param[out] error
 694  *   Pointer to error structure.
 695  *
 696  * @return
 697  *   0 on success, a negative errno value otherwise and rte_errno is set.
 698  */
 699 int
 700 mlx5_flow_validate_action_flag(uint64_t action_flags,
 701                                const struct rte_flow_attr *attr,
 702                                struct rte_flow_error *error)
 703 {
 704
 705         if (action_flags & MLX5_FLOW_ACTION_DROP)
 706                 return rte_flow_error_set(error, EINVAL,
 707                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 708                                           "can't drop and flag in same flow");
 709         if (action_flags & MLX5_FLOW_ACTION_MARK)
 710                 return rte_flow_error_set(error, EINVAL,
 711                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 712                                           "can't mark and flag in same flow");
 713         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 714                 return rte_flow_error_set(error, EINVAL,
 715                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 716                                           "can't have 2 flag"
 717                                           " actions in same flow");
 718         if (attr->egress)
 719                 return rte_flow_error_set(error, ENOTSUP,
 720                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 721                                           "flag action not supported for "
 722                                           "egress");
 723         return 0;
 724 }
 725
 726 /*
 727  * Validate the mark action.
 728  *
 729  * @param[in] action
 730  *   Pointer to the queue action.
 731  * @param[in] action_flags
 732  *   Bit-fields that holds the actions detected until now.
 733  * @param[in] attr
 734  *   Attributes of flow that includes this action.
 735  * @param[out] error
 736  *   Pointer to error structure.
 737  *
 738  * @return
 739  *   0 on success, a negative errno value otherwise and rte_errno is set.
 740  */
 741 int
 742 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
 743                                uint64_t action_flags,
 744                                const struct rte_flow_attr *attr,
 745                                struct rte_flow_error *error)
 746 {
 747         const struct rte_flow_action_mark *mark = action->conf;
 748
 749         if (!mark)
 750                 return rte_flow_error_set(error, EINVAL,
 751                                           RTE_FLOW_ERROR_TYPE_ACTION,
 752                                           action,
 753                                           "configuration cannot be null");
 754         if (mark->id >= MLX5_FLOW_MARK_MAX)
 755                 return rte_flow_error_set(error, EINVAL,
 756                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 757                                           &mark->id,
 758                                           "mark id must in 0 <= id < "
 759                                           RTE_STR(MLX5_FLOW_MARK_MAX));
 760         if (action_flags & MLX5_FLOW_ACTION_DROP)
 761                 return rte_flow_error_set(error, EINVAL,
 762                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 763                                           "can't drop and mark in same flow");
 764         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 765                 return rte_flow_error_set(error, EINVAL,
 766                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 767                                           "can't flag and mark in same flow");
 768         if (action_flags & MLX5_FLOW_ACTION_MARK)
 769                 return rte_flow_error_set(error, EINVAL,
 770                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 771                                           "can't have 2 mark actions in same"
 772                                           " flow");
 773         if (attr->egress)
 774                 return rte_flow_error_set(error, ENOTSUP,
 775                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 776                                           "mark action not supported for "
 777                                           "egress");
 778         return 0;
 779 }
 780
 781 /*
 782  * Validate the drop action.
 783  *
 784  * @param[in] action_flags
 785  *   Bit-fields that holds the actions detected until now.
 786  * @param[in] attr
 787  *   Attributes of flow that includes this action.
 788  * @param[out] error
 789  *   Pointer to error structure.
 790  *
 791  * @return
 792  *   0 on success, a negative errno value otherwise and rte_errno is set.
 793  */
 794 int
 795 mlx5_flow_validate_action_drop(uint64_t action_flags,
 796                                const struct rte_flow_attr *attr,
 797                                struct rte_flow_error *error)
 798 {
 799         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 800                 return rte_flow_error_set(error, EINVAL,
 801                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 802                                           "can't drop and flag in same flow");
 803         if (action_flags & MLX5_FLOW_ACTION_MARK)
 804                 return rte_flow_error_set(error, EINVAL,
 805                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 806                                           "can't drop and mark in same flow");
 807         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 808                 return rte_flow_error_set(error, EINVAL,
 809                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 810                                           "can't have 2 fate actions in"
 811                                           " same flow");
 812         if (attr->egress)
 813                 return rte_flow_error_set(error, ENOTSUP,
 814                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 815                                           "drop action not supported for "
 816                                           "egress");
 817         return 0;
 818 }
 819
 820 /*
 821  * Validate the queue action.
 822  *
 823  * @param[in] action
 824  *   Pointer to the queue action.
 825  * @param[in] action_flags
 826  *   Bit-fields that holds the actions detected until now.
 827  * @param[in] dev
 828  *   Pointer to the Ethernet device structure.
 829  * @param[in] attr
 830  *   Attributes of flow that includes this action.
 831  * @param[out] error
 832  *   Pointer to error structure.
 833  *
 834  * @return
 835  *   0 on success, a negative errno value otherwise and rte_errno is set.
 836  */
 837 int
 838 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
 839                                 uint64_t action_flags,
 840                                 struct rte_eth_dev *dev,
 841                                 const struct rte_flow_attr *attr,
 842                                 struct rte_flow_error *error)
 843 {
 844         struct mlx5_priv *priv = dev->data->dev_private;
 845         const struct rte_flow_action_queue *queue = action->conf;
 846
 847         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 848                 return rte_flow_error_set(error, EINVAL,
 849                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 850                                           "can't have 2 fate actions in"
 851                                           " same flow");
 852         if (!priv->rxqs_n)
 853                 return rte_flow_error_set(error, EINVAL,
 854                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 855                                           NULL, "No Rx queues configured");
 856         if (queue->index >= priv->rxqs_n)
 857                 return rte_flow_error_set(error, EINVAL,
 858                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 859                                           &queue->index,
 860                                           "queue index out of range");
 861         if (!(*priv->rxqs)[queue->index])
 862                 return rte_flow_error_set(error, EINVAL,
 863                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 864                                           &queue->index,
 865                                           "queue is not configured");
 866         if (attr->egress)
 867                 return rte_flow_error_set(error, ENOTSUP,
 868                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 869                                           "queue action not supported for "
 870                                           "egress");
 871         return 0;
 872 }
 873
 874 /*
 875  * Validate the rss action.
 876  *
 877  * @param[in] action
 878  *   Pointer to the queue action.
 879  * @param[in] action_flags
 880  *   Bit-fields that holds the actions detected until now.
 881  * @param[in] dev
 882  *   Pointer to the Ethernet device structure.
 883  * @param[in] attr
 884  *   Attributes of flow that includes this action.
 885  * @param[out] error
 886  *   Pointer to error structure.
 887  *
 888  * @return
 889  *   0 on success, a negative errno value otherwise and rte_errno is set.
 890  */
 891 int
 892 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
 893                               uint64_t action_flags,
 894                               struct rte_eth_dev *dev,
 895                               const struct rte_flow_attr *attr,
 896                               struct rte_flow_error *error)
 897 {
 898         struct mlx5_priv *priv = dev->data->dev_private;
 899         const struct rte_flow_action_rss *rss = action->conf;
 900         unsigned int i;
 901
 902         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 903                 return rte_flow_error_set(error, EINVAL,
 904                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 905                                           "can't have 2 fate actions"
 906                                           " in same flow");
 907         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
 908             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 909                 return rte_flow_error_set(error, ENOTSUP,
 910                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 911                                           &rss->func,
 912                                           "RSS hash function not supported");
 913 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 914         if (rss->level > 2)
 915 #else
 916         if (rss->level > 1)
 917 #endif
 918                 return rte_flow_error_set(error, ENOTSUP,
 919                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 920                                           &rss->level,
 921                                           "tunnel RSS is not supported");
 922         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
 923         if (rss->key_len == 0 && rss->key != NULL)
 924                 return rte_flow_error_set(error, ENOTSUP,
 925                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 926                                           &rss->key_len,
 927                                           "RSS hash key length 0");
 928         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
 929                 return rte_flow_error_set(error, ENOTSUP,
 930                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 931                                           &rss->key_len,
 932                                           "RSS hash key too small");
 933         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
 934                 return rte_flow_error_set(error, ENOTSUP,
 935                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 936                                           &rss->key_len,
 937                                           "RSS hash key too large");
 938         if (rss->queue_num > priv->config.ind_table_max_size)
 939                 return rte_flow_error_set(error, ENOTSUP,
 940                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 941                                           &rss->queue_num,
 942                                           "number of queues too large");
 943         if (rss->types & MLX5_RSS_HF_MASK)
 944                 return rte_flow_error_set(error, ENOTSUP,
 945                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 946                                           &rss->types,
 947                                           "some RSS protocols are not"
 948                                           " supported");
 949         if (!priv->rxqs_n)
 950                 return rte_flow_error_set(error, EINVAL,
 951                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 952                                           NULL, "No Rx queues configured");
 953         if (!rss->queue_num)
 954                 return rte_flow_error_set(error, EINVAL,
 955                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 956                                           NULL, "No queues configured");
 957         for (i = 0; i != rss->queue_num; ++i) {
 958                 if (!(*priv->rxqs)[rss->queue[i]])
 959                         return rte_flow_error_set
 960                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 961                                  &rss->queue[i], "queue is not configured");
 962         }
 963         if (attr->egress)
 964                 return rte_flow_error_set(error, ENOTSUP,
 965                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 966                                           "rss action not supported for "
 967                                           "egress");
 968         return 0;
 969 }
 970
 971 /*
 972  * Validate the count action.
 973  *
 974  * @param[in] dev
 975  *   Pointer to the Ethernet device structure.
 976  * @param[in] attr
 977  *   Attributes of flow that includes this action.
 978  * @param[out] error
 979  *   Pointer to error structure.
 980  *
 981  * @return
 982  *   0 on success, a negative errno value otherwise and rte_errno is set.
 983  */
 984 int
 985 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
 986                                 const struct rte_flow_attr *attr,
 987                                 struct rte_flow_error *error)
 988 {
 989         if (attr->egress)
 990                 return rte_flow_error_set(error, ENOTSUP,
 991                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 992                                           "count action not supported for "
 993                                           "egress");
 994         return 0;
 995 }
 996
 997 /**
 998  * Verify the @p attributes will be correctly understood by the NIC and store
 999  * them in the @p flow if everything is correct.
1000  *
1001  * @param[in] dev
1002  *   Pointer to the Ethernet device structure.
1003  * @param[in] attributes
1004  *   Pointer to flow attributes
1005  * @param[out] error
1006  *   Pointer to error structure.
1007  *
1008  * @return
1009  *   0 on success, a negative errno value otherwise and rte_errno is set.
1010  */
1011 int
1012 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1013                               const struct rte_flow_attr *attributes,
1014                               struct rte_flow_error *error)
1015 {
1016         struct mlx5_priv *priv = dev->data->dev_private;
1017         uint32_t priority_max = priv->config.flow_prio - 1;
1018
1019         if (attributes->group)
1020                 return rte_flow_error_set(error, ENOTSUP,
1021                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1022                                           NULL, "groups is not supported");
1023         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1024             attributes->priority >= priority_max)
1025                 return rte_flow_error_set(error, ENOTSUP,
1026                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1027                                           NULL, "priority out of range");
1028         if (attributes->egress)
1029                 return rte_flow_error_set(error, ENOTSUP,
1030                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1031                                           "egress is not supported");
1032         if (attributes->transfer)
1033                 return rte_flow_error_set(error, ENOTSUP,
1034                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1035                                           NULL, "transfer is not supported");
1036         if (!attributes->ingress)
1037                 return rte_flow_error_set(error, EINVAL,
1038                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1039                                           NULL,
1040                                           "ingress attribute is mandatory");
1041         return 0;
1042 }
1043
1044 /**
1045  * Validate Ethernet item.
1046  *
1047  * @param[in] item
1048  *   Item specification.
1049  * @param[in] item_flags
1050  *   Bit-fields that holds the items detected until now.
1051  * @param[out] error
1052  *   Pointer to error structure.
1053  *
1054  * @return
1055  *   0 on success, a negative errno value otherwise and rte_errno is set.
1056  */
1057 int
1058 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1059                             uint64_t item_flags,
1060                             struct rte_flow_error *error)
1061 {
1062         const struct rte_flow_item_eth *mask = item->mask;
1063         const struct rte_flow_item_eth nic_mask = {
1064                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1065                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1066                 .type = RTE_BE16(0xffff),
1067         };
1068         int ret;
1069         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1070         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1071                                        MLX5_FLOW_LAYER_OUTER_L2;
1072
1073         if (item_flags & ethm)
1074                 return rte_flow_error_set(error, ENOTSUP,
1075                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1076                                           "multiple L2 layers not supported");
1077         if (!mask)
1078                 mask = &rte_flow_item_eth_mask;
1079         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1080                                         (const uint8_t *)&nic_mask,
1081                                         sizeof(struct rte_flow_item_eth),
1082                                         error);
1083         return ret;
1084 }
1085
1086 /**
1087  * Validate VLAN item.
1088  *
1089  * @param[in] item
1090  *   Item specification.
1091  * @param[in] item_flags
1092  *   Bit-fields that holds the items detected until now.
1093  * @param[out] error
1094  *   Pointer to error structure.
1095  *
1096  * @return
1097  *   0 on success, a negative errno value otherwise and rte_errno is set.
1098  */
1099 int
1100 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1101                              uint64_t item_flags,
1102                              struct rte_flow_error *error)
1103 {
1104         const struct rte_flow_item_vlan *spec = item->spec;
1105         const struct rte_flow_item_vlan *mask = item->mask;
1106         const struct rte_flow_item_vlan nic_mask = {
1107                 .tci = RTE_BE16(0x0fff),
1108                 .inner_type = RTE_BE16(0xffff),
1109         };
1110         uint16_t vlan_tag = 0;
1111         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1112         int ret;
1113         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1114                                         MLX5_FLOW_LAYER_INNER_L4) :
1115                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1116                                         MLX5_FLOW_LAYER_OUTER_L4);
1117         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1118                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1119
1120         if (item_flags & vlanm)
1121                 return rte_flow_error_set(error, EINVAL,
1122                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1123                                           "multiple VLAN layers not supported");
1124         else if ((item_flags & l34m) != 0)
1125                 return rte_flow_error_set(error, EINVAL,
1126                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1127                                           "L2 layer cannot follow L3/L4 layer");
1128         if (!mask)
1129                 mask = &rte_flow_item_vlan_mask;
1130         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1131                                         (const uint8_t *)&nic_mask,
1132                                         sizeof(struct rte_flow_item_vlan),
1133                                         error);
1134         if (ret)
1135                 return ret;
1136         if (spec) {
1137                 vlan_tag = spec->tci;
1138                 vlan_tag &= mask->tci;
1139         }
1140         /*
1141          * From verbs perspective an empty VLAN is equivalent
1142          * to a packet without VLAN layer.
1143          */
1144         if (!vlan_tag)
1145                 return rte_flow_error_set(error, EINVAL,
1146                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1147                                           item->spec,
1148                                           "VLAN cannot be empty");
1149         return 0;
1150 }
1151
1152 /**
1153  * Validate IPV4 item.
1154  *
1155  * @param[in] item
1156  *   Item specification.
1157  * @param[in] item_flags
1158  *   Bit-fields that holds the items detected until now.
1159  * @param[in] acc_mask
1160  *   Acceptable mask, if NULL default internal default mask
1161  *   will be used to check whether item fields are supported.
1162  * @param[out] error
1163  *   Pointer to error structure.
1164  *
1165  * @return
1166  *   0 on success, a negative errno value otherwise and rte_errno is set.
1167  */
1168 int
1169 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1170                              uint64_t item_flags,
1171                              const struct rte_flow_item_ipv4 *acc_mask,
1172                              struct rte_flow_error *error)
1173 {
1174         const struct rte_flow_item_ipv4 *mask = item->mask;
1175         const struct rte_flow_item_ipv4 nic_mask = {
1176                 .hdr = {
1177                         .src_addr = RTE_BE32(0xffffffff),
1178                         .dst_addr = RTE_BE32(0xffffffff),
1179                         .type_of_service = 0xff,
1180                         .next_proto_id = 0xff,
1181                 },
1182         };
1183         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1184         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1185                                       MLX5_FLOW_LAYER_OUTER_L3;
1186         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1187                                       MLX5_FLOW_LAYER_OUTER_L4;
1188         int ret;
1189
1190         if (item_flags & l3m)
1191                 return rte_flow_error_set(error, ENOTSUP,
1192                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1193                                           "multiple L3 layers not supported");
1194         else if (item_flags & l4m)
1195                 return rte_flow_error_set(error, EINVAL,
1196                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1197                                           "L3 cannot follow an L4 layer.");
1198         if (!mask)
1199                 mask = &rte_flow_item_ipv4_mask;
1200         else if (mask->hdr.next_proto_id != 0 &&
1201                  mask->hdr.next_proto_id != 0xff)
1202                 return rte_flow_error_set(error, EINVAL,
1203                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1204                                           "partial mask is not supported"
1205                                           " for protocol");
1206         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1207                                         acc_mask ? (const uint8_t *)acc_mask
1208                                                  : (const uint8_t *)&nic_mask,
1209                                         sizeof(struct rte_flow_item_ipv4),
1210                                         error);
1211         if (ret < 0)
1212                 return ret;
1213         return 0;
1214 }
1215
1216 /**
1217  * Validate IPV6 item.
1218  *
1219  * @param[in] item
1220  *   Item specification.
1221  * @param[in] item_flags
1222  *   Bit-fields that holds the items detected until now.
1223  * @param[in] acc_mask
1224  *   Acceptable mask, if NULL default internal default mask
1225  *   will be used to check whether item fields are supported.
1226  * @param[out] error
1227  *   Pointer to error structure.
1228  *
1229  * @return
1230  *   0 on success, a negative errno value otherwise and rte_errno is set.
1231  */
1232 int
1233 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1234                              uint64_t item_flags,
1235                              const struct rte_flow_item_ipv6 *acc_mask,
1236                              struct rte_flow_error *error)
1237 {
1238         const struct rte_flow_item_ipv6 *mask = item->mask;
1239         const struct rte_flow_item_ipv6 nic_mask = {
1240                 .hdr = {
1241                         .src_addr =
1242                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1243                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1244                         .dst_addr =
1245                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1246                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1247                         .vtc_flow = RTE_BE32(0xffffffff),
1248                         .proto = 0xff,
1249                         .hop_limits = 0xff,
1250                 },
1251         };
1252         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1253         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1254                                       MLX5_FLOW_LAYER_OUTER_L3;
1255         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1256                                       MLX5_FLOW_LAYER_OUTER_L4;
1257         int ret;
1258
1259         if (item_flags & l3m)
1260                 return rte_flow_error_set(error, ENOTSUP,
1261                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1262                                           "multiple L3 layers not supported");
1263         else if (item_flags & l4m)
1264                 return rte_flow_error_set(error, EINVAL,
1265                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1266                                           "L3 cannot follow an L4 layer.");
1267         if (!mask)
1268                 mask = &rte_flow_item_ipv6_mask;
1269         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1270                                         acc_mask ? (const uint8_t *)acc_mask
1271                                                  : (const uint8_t *)&nic_mask,
1272                                         sizeof(struct rte_flow_item_ipv6),
1273                                         error);
1274         if (ret < 0)
1275                 return ret;
1276         return 0;
1277 }
1278
1279 /**
1280  * Validate UDP item.
1281  *
1282  * @param[in] item
1283  *   Item specification.
1284  * @param[in] item_flags
1285  *   Bit-fields that holds the items detected until now.
1286  * @param[in] target_protocol
1287  *   The next protocol in the previous item.
1288  * @param[in] flow_mask
1289  *   mlx5 flow-specific (TCF, DV, verbs, etc.) supported header fields mask.
1290  * @param[out] error
1291  *   Pointer to error structure.
1292  *
1293  * @return
1294  *   0 on success, a negative errno value otherwise and rte_errno is set.
1295  */
1296 int
1297 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1298                             uint64_t item_flags,
1299                             uint8_t target_protocol,
1300                             struct rte_flow_error *error)
1301 {
1302         const struct rte_flow_item_udp *mask = item->mask;
1303         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1304         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1305                                       MLX5_FLOW_LAYER_OUTER_L3;
1306         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1307                                       MLX5_FLOW_LAYER_OUTER_L4;
1308         int ret;
1309
1310         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1311                 return rte_flow_error_set(error, EINVAL,
1312                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1313                                           "protocol filtering not compatible"
1314                                           " with UDP layer");
1315         if (!(item_flags & l3m))
1316                 return rte_flow_error_set(error, EINVAL,
1317                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1318                                           "L3 is mandatory to filter on L4");
1319         if (item_flags & l4m)
1320                 return rte_flow_error_set(error, EINVAL,
1321                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1322                                           "multiple L4 layers not supported");
1323         if (!mask)
1324                 mask = &rte_flow_item_udp_mask;
1325         ret = mlx5_flow_item_acceptable
1326                 (item, (const uint8_t *)mask,
1327                  (const uint8_t *)&rte_flow_item_udp_mask,
1328                  sizeof(struct rte_flow_item_udp), error);
1329         if (ret < 0)
1330                 return ret;
1331         return 0;
1332 }
1333
1334 /**
1335  * Validate TCP item.
1336  *
1337  * @param[in] item
1338  *   Item specification.
1339  * @param[in] item_flags
1340  *   Bit-fields that holds the items detected until now.
1341  * @param[in] target_protocol
1342  *   The next protocol in the previous item.
1343  * @param[out] error
1344  *   Pointer to error structure.
1345  *
1346  * @return
1347  *   0 on success, a negative errno value otherwise and rte_errno is set.
1348  */
1349 int
1350 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1351                             uint64_t item_flags,
1352                             uint8_t target_protocol,
1353                             const struct rte_flow_item_tcp *flow_mask,
1354                             struct rte_flow_error *error)
1355 {
1356         const struct rte_flow_item_tcp *mask = item->mask;
1357         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1358         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1359                                       MLX5_FLOW_LAYER_OUTER_L3;
1360         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1361                                       MLX5_FLOW_LAYER_OUTER_L4;
1362         int ret;
1363
1364         assert(flow_mask);
1365         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1366                 return rte_flow_error_set(error, EINVAL,
1367                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1368                                           "protocol filtering not compatible"
1369                                           " with TCP layer");
1370         if (!(item_flags & l3m))
1371                 return rte_flow_error_set(error, EINVAL,
1372                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1373                                           "L3 is mandatory to filter on L4");
1374         if (item_flags & l4m)
1375                 return rte_flow_error_set(error, EINVAL,
1376                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1377                                           "multiple L4 layers not supported");
1378         if (!mask)
1379                 mask = &rte_flow_item_tcp_mask;
1380         ret = mlx5_flow_item_acceptable
1381                 (item, (const uint8_t *)mask,
1382                  (const uint8_t *)flow_mask,
1383                  sizeof(struct rte_flow_item_tcp), error);
1384         if (ret < 0)
1385                 return ret;
1386         return 0;
1387 }
1388
1389 /**
1390  * Validate VXLAN item.
1391  *
1392  * @param[in] item
1393  *   Item specification.
1394  * @param[in] item_flags
1395  *   Bit-fields that holds the items detected until now.
1396  * @param[in] target_protocol
1397  *   The next protocol in the previous item.
1398  * @param[out] error
1399  *   Pointer to error structure.
1400  *
1401  * @return
1402  *   0 on success, a negative errno value otherwise and rte_errno is set.
1403  */
1404 int
1405 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1406                               uint64_t item_flags,
1407                               struct rte_flow_error *error)
1408 {
1409         const struct rte_flow_item_vxlan *spec = item->spec;
1410         const struct rte_flow_item_vxlan *mask = item->mask;
1411         int ret;
1412         union vni {
1413                 uint32_t vlan_id;
1414                 uint8_t vni[4];
1415         } id = { .vlan_id = 0, };
1416         uint32_t vlan_id = 0;
1417
1418
1419         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1420                 return rte_flow_error_set(error, ENOTSUP,
1421                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1422                                           "multiple tunnel layers not"
1423                                           " supported");
1424         /*
1425          * Verify only UDPv4 is present as defined in
1426          * https://tools.ietf.org/html/rfc7348
1427          */
1428         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1429                 return rte_flow_error_set(error, EINVAL,
1430                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1431                                           "no outer UDP layer found");
1432         if (!mask)
1433                 mask = &rte_flow_item_vxlan_mask;
1434         ret = mlx5_flow_item_acceptable
1435                 (item, (const uint8_t *)mask,
1436                  (const uint8_t *)&rte_flow_item_vxlan_mask,
1437                  sizeof(struct rte_flow_item_vxlan),
1438                  error);
1439         if (ret < 0)
1440                 return ret;
1441         if (spec) {
1442                 memcpy(&id.vni[1], spec->vni, 3);
1443                 vlan_id = id.vlan_id;
1444                 memcpy(&id.vni[1], mask->vni, 3);
1445                 vlan_id &= id.vlan_id;
1446         }
1447         /*
1448          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
1449          * only this layer is defined in the Verbs specification it is
1450          * interpreted as wildcard and all packets will match this
1451          * rule, if it follows a full stack layer (ex: eth / ipv4 /
1452          * udp), all packets matching the layers before will also
1453          * match this rule.  To avoid such situation, VNI 0 is
1454          * currently refused.
1455          */
1456         if (!vlan_id)
1457                 return rte_flow_error_set(error, ENOTSUP,
1458                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1459                                           "VXLAN vni cannot be 0");
1460         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1461                 return rte_flow_error_set(error, ENOTSUP,
1462                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1463                                           "VXLAN tunnel must be fully defined");
1464         return 0;
1465 }
1466
1467 /**
1468  * Validate VXLAN_GPE item.
1469  *
1470  * @param[in] item
1471  *   Item specification.
1472  * @param[in] item_flags
1473  *   Bit-fields that holds the items detected until now.
1474  * @param[in] priv
1475  *   Pointer to the private data structure.
1476  * @param[in] target_protocol
1477  *   The next protocol in the previous item.
1478  * @param[out] error
1479  *   Pointer to error structure.
1480  *
1481  * @return
1482  *   0 on success, a negative errno value otherwise and rte_errno is set.
1483  */
1484 int
1485 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1486                                   uint64_t item_flags,
1487                                   struct rte_eth_dev *dev,
1488                                   struct rte_flow_error *error)
1489 {
1490         struct mlx5_priv *priv = dev->data->dev_private;
1491         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1492         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1493         int ret;
1494         union vni {
1495                 uint32_t vlan_id;
1496                 uint8_t vni[4];
1497         } id = { .vlan_id = 0, };
1498         uint32_t vlan_id = 0;
1499
1500         if (!priv->config.l3_vxlan_en)
1501                 return rte_flow_error_set(error, ENOTSUP,
1502                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1503                                           "L3 VXLAN is not enabled by device"
1504                                           " parameter and/or not configured in"
1505                                           " firmware");
1506         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1507                 return rte_flow_error_set(error, ENOTSUP,
1508                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1509                                           "multiple tunnel layers not"
1510                                           " supported");
1511         /*
1512          * Verify only UDPv4 is present as defined in
1513          * https://tools.ietf.org/html/rfc7348
1514          */
1515         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1516                 return rte_flow_error_set(error, EINVAL,
1517                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1518                                           "no outer UDP layer found");
1519         if (!mask)
1520                 mask = &rte_flow_item_vxlan_gpe_mask;
1521         ret = mlx5_flow_item_acceptable
1522                 (item, (const uint8_t *)mask,
1523                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1524                  sizeof(struct rte_flow_item_vxlan_gpe),
1525                  error);
1526         if (ret < 0)
1527                 return ret;
1528         if (spec) {
1529                 if (spec->protocol)
1530                         return rte_flow_error_set(error, ENOTSUP,
1531                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1532                                                   item,
1533                                                   "VxLAN-GPE protocol"
1534                                                   " not supported");
1535                 memcpy(&id.vni[1], spec->vni, 3);
1536                 vlan_id = id.vlan_id;
1537                 memcpy(&id.vni[1], mask->vni, 3);
1538                 vlan_id &= id.vlan_id;
1539         }
1540         /*
1541          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1542          * layer is defined in the Verbs specification it is interpreted as
1543          * wildcard and all packets will match this rule, if it follows a full
1544          * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1545          * before will also match this rule.  To avoid such situation, VNI 0
1546          * is currently refused.
1547          */
1548         if (!vlan_id)
1549                 return rte_flow_error_set(error, ENOTSUP,
1550                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1551                                           "VXLAN-GPE vni cannot be 0");
1552         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1553                 return rte_flow_error_set(error, ENOTSUP,
1554                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1555                                           "VXLAN-GPE tunnel must be fully"
1556                                           " defined");
1557         return 0;
1558 }
1559
1560 /**
1561  * Validate GRE item.
1562  *
1563  * @param[in] item
1564  *   Item specification.
1565  * @param[in] item_flags
1566  *   Bit flags to mark detected items.
1567  * @param[in] target_protocol
1568  *   The next protocol in the previous item.
1569  * @param[out] error
1570  *   Pointer to error structure.
1571  *
1572  * @return
1573  *   0 on success, a negative errno value otherwise and rte_errno is set.
1574  */
1575 int
1576 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
1577                             uint64_t item_flags,
1578                             uint8_t target_protocol,
1579                             struct rte_flow_error *error)
1580 {
1581         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
1582         const struct rte_flow_item_gre *mask = item->mask;
1583         int ret;
1584
1585         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
1586                 return rte_flow_error_set(error, EINVAL,
1587                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1588                                           "protocol filtering not compatible"
1589                                           " with this GRE layer");
1590         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1591                 return rte_flow_error_set(error, ENOTSUP,
1592                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1593                                           "multiple tunnel layers not"
1594                                           " supported");
1595         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1596                 return rte_flow_error_set(error, ENOTSUP,
1597                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1598                                           "L3 Layer is missing");
1599         if (!mask)
1600                 mask = &rte_flow_item_gre_mask;
1601         ret = mlx5_flow_item_acceptable
1602                 (item, (const uint8_t *)mask,
1603                  (const uint8_t *)&rte_flow_item_gre_mask,
1604                  sizeof(struct rte_flow_item_gre), error);
1605         if (ret < 0)
1606                 return ret;
1607 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
1608         if (spec && (spec->protocol & mask->protocol))
1609                 return rte_flow_error_set(error, ENOTSUP,
1610                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1611                                           "without MPLS support the"
1612                                           " specification cannot be used for"
1613                                           " filtering");
1614 #endif
1615         return 0;
1616 }
1617
1618 /**
1619  * Validate MPLS item.
1620  *
1621  * @param[in] dev
1622  *   Pointer to the rte_eth_dev structure.
1623  * @param[in] item
1624  *   Item specification.
1625  * @param[in] item_flags
1626  *   Bit-fields that holds the items detected until now.
1627  * @param[in] prev_layer
1628  *   The protocol layer indicated in previous item.
1629  * @param[out] error
1630  *   Pointer to error structure.
1631  *
1632  * @return
1633  *   0 on success, a negative errno value otherwise and rte_errno is set.
1634  */
1635 int
1636 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
1637                              const struct rte_flow_item *item __rte_unused,
1638                              uint64_t item_flags __rte_unused,
1639                              uint64_t prev_layer __rte_unused,
1640                              struct rte_flow_error *error)
1641 {
1642 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1643         const struct rte_flow_item_mpls *mask = item->mask;
1644         struct mlx5_priv *priv = dev->data->dev_private;
1645         int ret;
1646
1647         if (!priv->config.mpls_en)
1648                 return rte_flow_error_set(error, ENOTSUP,
1649                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1650                                           "MPLS not supported or"
1651                                           " disabled in firmware"
1652                                           " configuration.");
1653         /* MPLS over IP, UDP, GRE is allowed */
1654         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
1655                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
1656                             MLX5_FLOW_LAYER_GRE)))
1657                 return rte_flow_error_set(error, EINVAL,
1658                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1659                                           "protocol filtering not compatible"
1660                                           " with MPLS layer");
1661         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
1662         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1663             !(item_flags & MLX5_FLOW_LAYER_GRE))
1664                 return rte_flow_error_set(error, ENOTSUP,
1665                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1666                                           "multiple tunnel layers not"
1667                                           " supported");
1668         if (!mask)
1669                 mask = &rte_flow_item_mpls_mask;
1670         ret = mlx5_flow_item_acceptable
1671                 (item, (const uint8_t *)mask,
1672                  (const uint8_t *)&rte_flow_item_mpls_mask,
1673                  sizeof(struct rte_flow_item_mpls), error);
1674         if (ret < 0)
1675                 return ret;
1676         return 0;
1677 #endif
1678         return rte_flow_error_set(error, ENOTSUP,
1679                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
1680                                   "MPLS is not supported by Verbs, please"
1681                                   " update.");
1682 }
1683
1684 static int
1685 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
1686                    const struct rte_flow_attr *attr __rte_unused,
1687                    const struct rte_flow_item items[] __rte_unused,
1688                    const struct rte_flow_action actions[] __rte_unused,
1689                    struct rte_flow_error *error __rte_unused)
1690 {
1691         rte_errno = ENOTSUP;
1692         return -rte_errno;
1693 }
1694
1695 static struct mlx5_flow *
1696 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
1697                   const struct rte_flow_item items[] __rte_unused,
1698                   const struct rte_flow_action actions[] __rte_unused,
1699                   struct rte_flow_error *error __rte_unused)
1700 {
1701         rte_errno = ENOTSUP;
1702         return NULL;
1703 }
1704
1705 static int
1706 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
1707                     struct mlx5_flow *dev_flow __rte_unused,
1708                     const struct rte_flow_attr *attr __rte_unused,
1709                     const struct rte_flow_item items[] __rte_unused,
1710                     const struct rte_flow_action actions[] __rte_unused,
1711                     struct rte_flow_error *error __rte_unused)
1712 {
1713         rte_errno = ENOTSUP;
1714         return -rte_errno;
1715 }
1716
1717 static int
1718 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
1719                 struct rte_flow *flow __rte_unused,
1720                 struct rte_flow_error *error __rte_unused)
1721 {
1722         rte_errno = ENOTSUP;
1723         return -rte_errno;
1724 }
1725
1726 static void
1727 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
1728                  struct rte_flow *flow __rte_unused)
1729 {
1730 }
1731
1732 static void
1733 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
1734                   struct rte_flow *flow __rte_unused)
1735 {
1736 }
1737
1738 static int
1739 flow_null_query(struct rte_eth_dev *dev __rte_unused,
1740                 struct rte_flow *flow __rte_unused,
1741                 const struct rte_flow_action *actions __rte_unused,
1742                 void *data __rte_unused,
1743                 struct rte_flow_error *error __rte_unused)
1744 {
1745         rte_errno = ENOTSUP;
1746         return -rte_errno;
1747 }
1748
1749 /* Void driver to protect from null pointer reference. */
1750 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
1751         .validate = flow_null_validate,
1752         .prepare = flow_null_prepare,
1753         .translate = flow_null_translate,
1754         .apply = flow_null_apply,
1755         .remove = flow_null_remove,
1756         .destroy = flow_null_destroy,
1757         .query = flow_null_query,
1758 };
1759
1760 /**
1761  * Select flow driver type according to flow attributes and device
1762  * configuration.
1763  *
1764  * @param[in] dev
1765  *   Pointer to the dev structure.
1766  * @param[in] attr
1767  *   Pointer to the flow attributes.
1768  *
1769  * @return
1770  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
1771  */
1772 static enum mlx5_flow_drv_type
1773 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
1774 {
1775         struct mlx5_priv *priv = dev->data->dev_private;
1776         enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
1777
1778         if (attr->transfer)
1779                 type = MLX5_FLOW_TYPE_TCF;
1780         else
1781                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
1782                                                  MLX5_FLOW_TYPE_VERBS;
1783         return type;
1784 }
1785
1786 #define flow_get_drv_ops(type) flow_drv_ops[type]
1787
1788 /**
1789  * Flow driver validation API. This abstracts calling driver specific functions.
1790  * The type of flow driver is determined according to flow attributes.
1791  *
1792  * @param[in] dev
1793  *   Pointer to the dev structure.
1794  * @param[in] attr
1795  *   Pointer to the flow attributes.
1796  * @param[in] items
1797  *   Pointer to the list of items.
1798  * @param[in] actions
1799  *   Pointer to the list of actions.
1800  * @param[out] error
1801  *   Pointer to the error structure.
1802  *
1803  * @return
1804  *   0 on success, a negative errno value otherwise and rte_errno is set.
1805  */
1806 static inline int
1807 flow_drv_validate(struct rte_eth_dev *dev,
1808                   const struct rte_flow_attr *attr,
1809                   const struct rte_flow_item items[],
1810                   const struct rte_flow_action actions[],
1811                   struct rte_flow_error *error)
1812 {
1813         const struct mlx5_flow_driver_ops *fops;
1814         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
1815
1816         fops = flow_get_drv_ops(type);
1817         return fops->validate(dev, attr, items, actions, error);
1818 }
1819
1820 /**
1821  * Flow driver preparation API. This abstracts calling driver specific
1822  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1823  * calculates the size of memory required for device flow, allocates the memory,
1824  * initializes the device flow and returns the pointer.
1825  *
1826  * @note
1827  *   This function initializes device flow structure such as dv, tcf or verbs in
1828  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
1829  *   rest. For example, adding returning device flow to flow->dev_flow list and
1830  *   setting backward reference to the flow should be done out of this function.
1831  *   layers field is not filled either.
1832  *
1833  * @param[in] attr
1834  *   Pointer to the flow attributes.
1835  * @param[in] items
1836  *   Pointer to the list of items.
1837  * @param[in] actions
1838  *   Pointer to the list of actions.
1839  * @param[out] error
1840  *   Pointer to the error structure.
1841  *
1842  * @return
1843  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
1844  */
1845 static inline struct mlx5_flow *
1846 flow_drv_prepare(const struct rte_flow *flow,
1847                  const struct rte_flow_attr *attr,
1848                  const struct rte_flow_item items[],
1849                  const struct rte_flow_action actions[],
1850                  struct rte_flow_error *error)
1851 {
1852         const struct mlx5_flow_driver_ops *fops;
1853         enum mlx5_flow_drv_type type = flow->drv_type;
1854
1855         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1856         fops = flow_get_drv_ops(type);
1857         return fops->prepare(attr, items, actions, error);
1858 }
1859
1860 /**
1861  * Flow driver translation API. This abstracts calling driver specific
1862  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1863  * translates a generic flow into a driver flow. flow_drv_prepare() must
1864  * precede.
1865  *
1866  * @note
1867  *   dev_flow->layers could be filled as a result of parsing during translation
1868  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
1869  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
1870  *   flow->actions could be overwritten even though all the expanded dev_flows
1871  *   have the same actions.
1872  *
1873  * @param[in] dev
1874  *   Pointer to the rte dev structure.
1875  * @param[in, out] dev_flow
1876  *   Pointer to the mlx5 flow.
1877  * @param[in] attr
1878  *   Pointer to the flow attributes.
1879  * @param[in] items
1880  *   Pointer to the list of items.
1881  * @param[in] actions
1882  *   Pointer to the list of actions.
1883  * @param[out] error
1884  *   Pointer to the error structure.
1885  *
1886  * @return
1887  *   0 on success, a negative errno value otherwise and rte_errno is set.
1888  */
1889 static inline int
1890 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
1891                    const struct rte_flow_attr *attr,
1892                    const struct rte_flow_item items[],
1893                    const struct rte_flow_action actions[],
1894                    struct rte_flow_error *error)
1895 {
1896         const struct mlx5_flow_driver_ops *fops;
1897         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
1898
1899         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1900         fops = flow_get_drv_ops(type);
1901         return fops->translate(dev, dev_flow, attr, items, actions, error);
1902 }
1903
1904 /**
1905  * Flow driver apply API. This abstracts calling driver specific functions.
1906  * Parent flow (rte_flow) should have driver type (drv_type). It applies
1907  * translated driver flows on to device. flow_drv_translate() must precede.
1908  *
1909  * @param[in] dev
1910  *   Pointer to Ethernet device structure.
1911  * @param[in, out] flow
1912  *   Pointer to flow structure.
1913  * @param[out] error
1914  *   Pointer to error structure.
1915  *
1916  * @return
1917  *   0 on success, a negative errno value otherwise and rte_errno is set.
1918  */
1919 static inline int
1920 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1921                struct rte_flow_error *error)
1922 {
1923         const struct mlx5_flow_driver_ops *fops;
1924         enum mlx5_flow_drv_type type = flow->drv_type;
1925
1926         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1927         fops = flow_get_drv_ops(type);
1928         return fops->apply(dev, flow, error);
1929 }
1930
1931 /**
1932  * Flow driver remove API. This abstracts calling driver specific functions.
1933  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1934  * on device. All the resources of the flow should be freed by calling
1935  * flow_drv_destroy().
1936  *
1937  * @param[in] dev
1938  *   Pointer to Ethernet device.
1939  * @param[in, out] flow
1940  *   Pointer to flow structure.
1941  */
1942 static inline void
1943 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1944 {
1945         const struct mlx5_flow_driver_ops *fops;
1946         enum mlx5_flow_drv_type type = flow->drv_type;
1947
1948         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1949         fops = flow_get_drv_ops(type);
1950         fops->remove(dev, flow);
1951 }
1952
1953 /**
1954  * Flow driver destroy API. This abstracts calling driver specific functions.
1955  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1956  * on device and releases resources of the flow.
1957  *
1958  * @param[in] dev
1959  *   Pointer to Ethernet device.
1960  * @param[in, out] flow
1961  *   Pointer to flow structure.
1962  */
1963 static inline void
1964 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1965 {
1966         const struct mlx5_flow_driver_ops *fops;
1967         enum mlx5_flow_drv_type type = flow->drv_type;
1968
1969         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1970         fops = flow_get_drv_ops(type);
1971         fops->destroy(dev, flow);
1972 }
1973
1974 /**
1975  * Validate a flow supported by the NIC.
1976  *
1977  * @see rte_flow_validate()
1978  * @see rte_flow_ops
1979  */
1980 int
1981 mlx5_flow_validate(struct rte_eth_dev *dev,
1982                    const struct rte_flow_attr *attr,
1983                    const struct rte_flow_item items[],
1984                    const struct rte_flow_action actions[],
1985                    struct rte_flow_error *error)
1986 {
1987         int ret;
1988
1989         ret = flow_drv_validate(dev, attr, items, actions, error);
1990         if (ret < 0)
1991                 return ret;
1992         return 0;
1993 }
1994
1995 /**
1996  * Get RSS action from the action list.
1997  *
1998  * @param[in] actions
1999  *   Pointer to the list of actions.
2000  *
2001  * @return
2002  *   Pointer to the RSS action if exist, else return NULL.
2003  */
2004 static const struct rte_flow_action_rss*
2005 flow_get_rss_action(const struct rte_flow_action actions[])
2006 {
2007         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2008                 switch (actions->type) {
2009                 case RTE_FLOW_ACTION_TYPE_RSS:
2010                         return (const struct rte_flow_action_rss *)
2011                                actions->conf;
2012                 default:
2013                         break;
2014                 }
2015         }
2016         return NULL;
2017 }
2018
2019 static unsigned int
2020 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
2021 {
2022         const struct rte_flow_item *item;
2023         unsigned int has_vlan = 0;
2024
2025         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2026                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
2027                         has_vlan = 1;
2028                         break;
2029                 }
2030         }
2031         if (has_vlan)
2032                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
2033                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
2034         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
2035                                MLX5_EXPANSION_ROOT_OUTER;
2036 }
2037
2038 /**
2039  * Create a flow and add it to @p list.
2040  *
2041  * @param dev
2042  *   Pointer to Ethernet device.
2043  * @param list
2044  *   Pointer to a TAILQ flow list.
2045  * @param[in] attr
2046  *   Flow rule attributes.
2047  * @param[in] items
2048  *   Pattern specification (list terminated by the END pattern item).
2049  * @param[in] actions
2050  *   Associated actions (list terminated by the END action).
2051  * @param[out] error
2052  *   Perform verbose error reporting if not NULL.
2053  *
2054  * @return
2055  *   A flow on success, NULL otherwise and rte_errno is set.
2056  */
2057 static struct rte_flow *
2058 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
2059                  const struct rte_flow_attr *attr,
2060                  const struct rte_flow_item items[],
2061                  const struct rte_flow_action actions[],
2062                  struct rte_flow_error *error)
2063 {
2064         struct rte_flow *flow = NULL;
2065         struct mlx5_flow *dev_flow;
2066         const struct rte_flow_action_rss *rss;
2067         union {
2068                 struct rte_flow_expand_rss buf;
2069                 uint8_t buffer[2048];
2070         } expand_buffer;
2071         struct rte_flow_expand_rss *buf = &expand_buffer.buf;
2072         int ret;
2073         uint32_t i;
2074         uint32_t flow_size;
2075
2076         ret = flow_drv_validate(dev, attr, items, actions, error);
2077         if (ret < 0)
2078                 return NULL;
2079         flow_size = sizeof(struct rte_flow);
2080         rss = flow_get_rss_action(actions);
2081         if (rss)
2082                 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
2083                                             sizeof(void *));
2084         else
2085                 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
2086         flow = rte_calloc(__func__, 1, flow_size, 0);
2087         flow->drv_type = flow_get_drv_type(dev, attr);
2088         flow->ingress = attr->ingress;
2089         assert(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
2090                flow->drv_type < MLX5_FLOW_TYPE_MAX);
2091         flow->queue = (void *)(flow + 1);
2092         LIST_INIT(&flow->dev_flows);
2093         if (rss && rss->types) {
2094                 unsigned int graph_root;
2095
2096                 graph_root = find_graph_root(items, rss->level);
2097                 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
2098                                           items, rss->types,
2099                                           mlx5_support_expansion,
2100                                           graph_root);
2101                 assert(ret > 0 &&
2102                        (unsigned int)ret < sizeof(expand_buffer.buffer));
2103         } else {
2104                 buf->entries = 1;
2105                 buf->entry[0].pattern = (void *)(uintptr_t)items;
2106         }
2107         for (i = 0; i < buf->entries; ++i) {
2108                 dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern,
2109                                             actions, error);
2110                 if (!dev_flow)
2111                         goto error;
2112                 dev_flow->flow = flow;
2113                 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
2114                 ret = flow_drv_translate(dev, dev_flow, attr,
2115                                          buf->entry[i].pattern,
2116                                          actions, error);
2117                 if (ret < 0)
2118                         goto error;
2119         }
2120         if (dev->data->dev_started) {
2121                 ret = flow_drv_apply(dev, flow, error);
2122                 if (ret < 0)
2123                         goto error;
2124         }
2125         TAILQ_INSERT_TAIL(list, flow, next);
2126         flow_rxq_flags_set(dev, flow);
2127         return flow;
2128 error:
2129         ret = rte_errno; /* Save rte_errno before cleanup. */
2130         assert(flow);
2131         flow_drv_destroy(dev, flow);
2132         rte_free(flow);
2133         rte_errno = ret; /* Restore rte_errno. */
2134         return NULL;
2135 }
2136
2137 /**
2138  * Create a flow.
2139  *
2140  * @see rte_flow_create()
2141  * @see rte_flow_ops
2142  */
2143 struct rte_flow *
2144 mlx5_flow_create(struct rte_eth_dev *dev,
2145                  const struct rte_flow_attr *attr,
2146                  const struct rte_flow_item items[],
2147                  const struct rte_flow_action actions[],
2148                  struct rte_flow_error *error)
2149 {
2150         struct mlx5_priv *priv = (struct mlx5_priv *)dev->data->dev_private;
2151
2152         return flow_list_create(dev, &priv->flows,
2153                                 attr, items, actions, error);
2154 }
2155
2156 /**
2157  * Destroy a flow in a list.
2158  *
2159  * @param dev
2160  *   Pointer to Ethernet device.
2161  * @param list
2162  *   Pointer to a TAILQ flow list.
2163  * @param[in] flow
2164  *   Flow to destroy.
2165  */
2166 static void
2167 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
2168                   struct rte_flow *flow)
2169 {
2170         /*
2171          * Update RX queue flags only if port is started, otherwise it is
2172          * already clean.
2173          */
2174         if (dev->data->dev_started)
2175                 flow_rxq_flags_trim(dev, flow);
2176         flow_drv_destroy(dev, flow);
2177         TAILQ_REMOVE(list, flow, next);
2178         rte_free(flow->fdir);
2179         rte_free(flow);
2180 }
2181
2182 /**
2183  * Destroy all flows.
2184  *
2185  * @param dev
2186  *   Pointer to Ethernet device.
2187  * @param list
2188  *   Pointer to a TAILQ flow list.
2189  */
2190 void
2191 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
2192 {
2193         while (!TAILQ_EMPTY(list)) {
2194                 struct rte_flow *flow;
2195
2196                 flow = TAILQ_FIRST(list);
2197                 flow_list_destroy(dev, list, flow);
2198         }
2199 }
2200
2201 /**
2202  * Remove all flows.
2203  *
2204  * @param dev
2205  *   Pointer to Ethernet device.
2206  * @param list
2207  *   Pointer to a TAILQ flow list.
2208  */
2209 void
2210 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
2211 {
2212         struct rte_flow *flow;
2213
2214         TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
2215                 flow_drv_remove(dev, flow);
2216         flow_rxq_flags_clear(dev);
2217 }
2218
2219 /**
2220  * Add all flows.
2221  *
2222  * @param dev
2223  *   Pointer to Ethernet device.
2224  * @param list
2225  *   Pointer to a TAILQ flow list.
2226  *
2227  * @return
2228  *   0 on success, a negative errno value otherwise and rte_errno is set.
2229  */
2230 int
2231 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
2232 {
2233         struct rte_flow *flow;
2234         struct rte_flow_error error;
2235         int ret = 0;
2236
2237         TAILQ_FOREACH(flow, list, next) {
2238                 ret = flow_drv_apply(dev, flow, &error);
2239                 if (ret < 0)
2240                         goto error;
2241                 flow_rxq_flags_set(dev, flow);
2242         }
2243         return 0;
2244 error:
2245         ret = rte_errno; /* Save rte_errno before cleanup. */
2246         mlx5_flow_stop(dev, list);
2247         rte_errno = ret; /* Restore rte_errno. */
2248         return -rte_errno;
2249 }
2250
2251 /**
2252  * Verify the flow list is empty
2253  *
2254  * @param dev
2255  *  Pointer to Ethernet device.
2256  *
2257  * @return the number of flows not released.
2258  */
2259 int
2260 mlx5_flow_verify(struct rte_eth_dev *dev)
2261 {
2262         struct mlx5_priv *priv = dev->data->dev_private;
2263         struct rte_flow *flow;
2264         int ret = 0;
2265
2266         TAILQ_FOREACH(flow, &priv->flows, next) {
2267                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
2268                         dev->data->port_id, (void *)flow);
2269                 ++ret;
2270         }
2271         return ret;
2272 }
2273
2274 /**
2275  * Enable a control flow configured from the control plane.
2276  *
2277  * @param dev
2278  *   Pointer to Ethernet device.
2279  * @param eth_spec
2280  *   An Ethernet flow spec to apply.
2281  * @param eth_mask
2282  *   An Ethernet flow mask to apply.
2283  * @param vlan_spec
2284  *   A VLAN flow spec to apply.
2285  * @param vlan_mask
2286  *   A VLAN flow mask to apply.
2287  *
2288  * @return
2289  *   0 on success, a negative errno value otherwise and rte_errno is set.
2290  */
2291 int
2292 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2293                     struct rte_flow_item_eth *eth_spec,
2294                     struct rte_flow_item_eth *eth_mask,
2295                     struct rte_flow_item_vlan *vlan_spec,
2296                     struct rte_flow_item_vlan *vlan_mask)
2297 {
2298         struct mlx5_priv *priv = dev->data->dev_private;
2299         const struct rte_flow_attr attr = {
2300                 .ingress = 1,
2301                 .priority = MLX5_FLOW_PRIO_RSVD,
2302         };
2303         struct rte_flow_item items[] = {
2304                 {
2305                         .type = RTE_FLOW_ITEM_TYPE_ETH,
2306                         .spec = eth_spec,
2307                         .last = NULL,
2308                         .mask = eth_mask,
2309                 },
2310                 {
2311                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2312                                               RTE_FLOW_ITEM_TYPE_END,
2313                         .spec = vlan_spec,
2314                         .last = NULL,
2315                         .mask = vlan_mask,
2316                 },
2317                 {
2318                         .type = RTE_FLOW_ITEM_TYPE_END,
2319                 },
2320         };
2321         uint16_t queue[priv->reta_idx_n];
2322         struct rte_flow_action_rss action_rss = {
2323                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
2324                 .level = 0,
2325                 .types = priv->rss_conf.rss_hf,
2326                 .key_len = priv->rss_conf.rss_key_len,
2327                 .queue_num = priv->reta_idx_n,
2328                 .key = priv->rss_conf.rss_key,
2329                 .queue = queue,
2330         };
2331         struct rte_flow_action actions[] = {
2332                 {
2333                         .type = RTE_FLOW_ACTION_TYPE_RSS,
2334                         .conf = &action_rss,
2335                 },
2336                 {
2337                         .type = RTE_FLOW_ACTION_TYPE_END,
2338                 },
2339         };
2340         struct rte_flow *flow;
2341         struct rte_flow_error error;
2342         unsigned int i;
2343
2344         if (!priv->reta_idx_n || !priv->rxqs_n) {
2345                 return 0;
2346         }
2347         for (i = 0; i != priv->reta_idx_n; ++i)
2348                 queue[i] = (*priv->reta_idx)[i];
2349         flow = flow_list_create(dev, &priv->ctrl_flows,
2350                                 &attr, items, actions, &error);
2351         if (!flow)
2352                 return -rte_errno;
2353         return 0;
2354 }
2355
2356 /**
2357  * Enable a flow control configured from the control plane.
2358  *
2359  * @param dev
2360  *   Pointer to Ethernet device.
2361  * @param eth_spec
2362  *   An Ethernet flow spec to apply.
2363  * @param eth_mask
2364  *   An Ethernet flow mask to apply.
2365  *
2366  * @return
2367  *   0 on success, a negative errno value otherwise and rte_errno is set.
2368  */
2369 int
2370 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2371                struct rte_flow_item_eth *eth_spec,
2372                struct rte_flow_item_eth *eth_mask)
2373 {
2374         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2375 }
2376
2377 /**
2378  * Destroy a flow.
2379  *
2380  * @see rte_flow_destroy()
2381  * @see rte_flow_ops
2382  */
2383 int
2384 mlx5_flow_destroy(struct rte_eth_dev *dev,
2385                   struct rte_flow *flow,
2386                   struct rte_flow_error *error __rte_unused)
2387 {
2388         struct mlx5_priv *priv = dev->data->dev_private;
2389
2390         flow_list_destroy(dev, &priv->flows, flow);
2391         return 0;
2392 }
2393
2394 /**
2395  * Destroy all flows.
2396  *
2397  * @see rte_flow_flush()
2398  * @see rte_flow_ops
2399  */
2400 int
2401 mlx5_flow_flush(struct rte_eth_dev *dev,
2402                 struct rte_flow_error *error __rte_unused)
2403 {
2404         struct mlx5_priv *priv = dev->data->dev_private;
2405
2406         mlx5_flow_list_flush(dev, &priv->flows);
2407         return 0;
2408 }
2409
2410 /**
2411  * Isolated mode.
2412  *
2413  * @see rte_flow_isolate()
2414  * @see rte_flow_ops
2415  */
2416 int
2417 mlx5_flow_isolate(struct rte_eth_dev *dev,
2418                   int enable,
2419                   struct rte_flow_error *error)
2420 {
2421         struct mlx5_priv *priv = dev->data->dev_private;
2422
2423         if (dev->data->dev_started) {
2424                 rte_flow_error_set(error, EBUSY,
2425                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2426                                    NULL,
2427                                    "port must be stopped first");
2428                 return -rte_errno;
2429         }
2430         priv->isolated = !!enable;
2431         if (enable)
2432                 dev->dev_ops = &mlx5_dev_ops_isolate;
2433         else
2434                 dev->dev_ops = &mlx5_dev_ops;
2435         return 0;
2436 }
2437
2438 /**
2439  * Query a flow.
2440  *
2441  * @see rte_flow_query()
2442  * @see rte_flow_ops
2443  */
2444 static int
2445 flow_drv_query(struct rte_eth_dev *dev,
2446                struct rte_flow *flow,
2447                const struct rte_flow_action *actions,
2448                void *data,
2449                struct rte_flow_error *error)
2450 {
2451         const struct mlx5_flow_driver_ops *fops;
2452         enum mlx5_flow_drv_type ftype = flow->drv_type;
2453
2454         assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
2455         fops = flow_get_drv_ops(ftype);
2456
2457         return fops->query(dev, flow, actions, data, error);
2458 }
2459
2460 /**
2461  * Query a flow.
2462  *
2463  * @see rte_flow_query()
2464  * @see rte_flow_ops
2465  */
2466 int
2467 mlx5_flow_query(struct rte_eth_dev *dev,
2468                 struct rte_flow *flow,
2469                 const struct rte_flow_action *actions,
2470                 void *data,
2471                 struct rte_flow_error *error)
2472 {
2473         int ret;
2474
2475         ret = flow_drv_query(dev, flow, actions, data, error);
2476         if (ret < 0)
2477                 return ret;
2478         return 0;
2479 }
2480
2481 /**
2482  * Convert a flow director filter to a generic flow.
2483  *
2484  * @param dev
2485  *   Pointer to Ethernet device.
2486  * @param fdir_filter
2487  *   Flow director filter to add.
2488  * @param attributes
2489  *   Generic flow parameters structure.
2490  *
2491  * @return
2492  *   0 on success, a negative errno value otherwise and rte_errno is set.
2493  */
2494 static int
2495 flow_fdir_filter_convert(struct rte_eth_dev *dev,
2496                          const struct rte_eth_fdir_filter *fdir_filter,
2497                          struct mlx5_fdir *attributes)
2498 {
2499         struct mlx5_priv *priv = dev->data->dev_private;
2500         const struct rte_eth_fdir_input *input = &fdir_filter->input;
2501         const struct rte_eth_fdir_masks *mask =
2502                 &dev->data->dev_conf.fdir_conf.mask;
2503
2504         /* Validate queue number. */
2505         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2506                 DRV_LOG(ERR, "port %u invalid queue number %d",
2507                         dev->data->port_id, fdir_filter->action.rx_queue);
2508                 rte_errno = EINVAL;
2509                 return -rte_errno;
2510         }
2511         attributes->attr.ingress = 1;
2512         attributes->items[0] = (struct rte_flow_item) {
2513                 .type = RTE_FLOW_ITEM_TYPE_ETH,
2514                 .spec = &attributes->l2,
2515                 .mask = &attributes->l2_mask,
2516         };
2517         switch (fdir_filter->action.behavior) {
2518         case RTE_ETH_FDIR_ACCEPT:
2519                 attributes->actions[0] = (struct rte_flow_action){
2520                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2521                         .conf = &attributes->queue,
2522                 };
2523                 break;
2524         case RTE_ETH_FDIR_REJECT:
2525                 attributes->actions[0] = (struct rte_flow_action){
2526                         .type = RTE_FLOW_ACTION_TYPE_DROP,
2527                 };
2528                 break;
2529         default:
2530                 DRV_LOG(ERR, "port %u invalid behavior %d",
2531                         dev->data->port_id,
2532                         fdir_filter->action.behavior);
2533                 rte_errno = ENOTSUP;
2534                 return -rte_errno;
2535         }
2536         attributes->queue.index = fdir_filter->action.rx_queue;
2537         /* Handle L3. */
2538         switch (fdir_filter->input.flow_type) {
2539         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2540         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2541         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2542                 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2543                         .src_addr = input->flow.ip4_flow.src_ip,
2544                         .dst_addr = input->flow.ip4_flow.dst_ip,
2545                         .time_to_live = input->flow.ip4_flow.ttl,
2546                         .type_of_service = input->flow.ip4_flow.tos,
2547                 };
2548                 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
2549                         .src_addr = mask->ipv4_mask.src_ip,
2550                         .dst_addr = mask->ipv4_mask.dst_ip,
2551                         .time_to_live = mask->ipv4_mask.ttl,
2552                         .type_of_service = mask->ipv4_mask.tos,
2553                         .next_proto_id = mask->ipv4_mask.proto,
2554                 };
2555                 attributes->items[1] = (struct rte_flow_item){
2556                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
2557                         .spec = &attributes->l3,
2558                         .mask = &attributes->l3_mask,
2559                 };
2560                 break;
2561         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2562         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2563         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2564                 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2565                         .hop_limits = input->flow.ipv6_flow.hop_limits,
2566                         .proto = input->flow.ipv6_flow.proto,
2567                 };
2568
2569                 memcpy(attributes->l3.ipv6.hdr.src_addr,
2570                        input->flow.ipv6_flow.src_ip,
2571                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2572                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2573                        input->flow.ipv6_flow.dst_ip,
2574                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2575                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
2576                        mask->ipv6_mask.src_ip,
2577                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2578                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
2579                        mask->ipv6_mask.dst_ip,
2580                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2581                 attributes->items[1] = (struct rte_flow_item){
2582                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
2583                         .spec = &attributes->l3,
2584                         .mask = &attributes->l3_mask,
2585                 };
2586                 break;
2587         default:
2588                 DRV_LOG(ERR, "port %u invalid flow type%d",
2589                         dev->data->port_id, fdir_filter->input.flow_type);
2590                 rte_errno = ENOTSUP;
2591                 return -rte_errno;
2592         }
2593         /* Handle L4. */
2594         switch (fdir_filter->input.flow_type) {
2595         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2596                 attributes->l4.udp.hdr = (struct udp_hdr){
2597                         .src_port = input->flow.udp4_flow.src_port,
2598                         .dst_port = input->flow.udp4_flow.dst_port,
2599                 };
2600                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2601                         .src_port = mask->src_port_mask,
2602                         .dst_port = mask->dst_port_mask,
2603                 };
2604                 attributes->items[2] = (struct rte_flow_item){
2605                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2606                         .spec = &attributes->l4,
2607                         .mask = &attributes->l4_mask,
2608                 };
2609                 break;
2610         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2611                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2612                         .src_port = input->flow.tcp4_flow.src_port,
2613                         .dst_port = input->flow.tcp4_flow.dst_port,
2614                 };
2615                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2616                         .src_port = mask->src_port_mask,
2617                         .dst_port = mask->dst_port_mask,
2618                 };
2619                 attributes->items[2] = (struct rte_flow_item){
2620                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2621                         .spec = &attributes->l4,
2622                         .mask = &attributes->l4_mask,
2623                 };
2624                 break;
2625         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2626                 attributes->l4.udp.hdr = (struct udp_hdr){
2627                         .src_port = input->flow.udp6_flow.src_port,
2628                         .dst_port = input->flow.udp6_flow.dst_port,
2629                 };
2630                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2631                         .src_port = mask->src_port_mask,
2632                         .dst_port = mask->dst_port_mask,
2633                 };
2634                 attributes->items[2] = (struct rte_flow_item){
2635                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2636                         .spec = &attributes->l4,
2637                         .mask = &attributes->l4_mask,
2638                 };
2639                 break;
2640         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2641                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2642                         .src_port = input->flow.tcp6_flow.src_port,
2643                         .dst_port = input->flow.tcp6_flow.dst_port,
2644                 };
2645                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2646                         .src_port = mask->src_port_mask,
2647                         .dst_port = mask->dst_port_mask,
2648                 };
2649                 attributes->items[2] = (struct rte_flow_item){
2650                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2651                         .spec = &attributes->l4,
2652                         .mask = &attributes->l4_mask,
2653                 };
2654                 break;
2655         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2656         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2657                 break;
2658         default:
2659                 DRV_LOG(ERR, "port %u invalid flow type%d",
2660                         dev->data->port_id, fdir_filter->input.flow_type);
2661                 rte_errno = ENOTSUP;
2662                 return -rte_errno;
2663         }
2664         return 0;
2665 }
2666
2667 #define FLOW_FDIR_CMP(f1, f2, fld) \
2668         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
2669
2670 /**
2671  * Compare two FDIR flows. If items and actions are identical, the two flows are
2672  * regarded as same.
2673  *
2674  * @param dev
2675  *   Pointer to Ethernet device.
2676  * @param f1
2677  *   FDIR flow to compare.
2678  * @param f2
2679  *   FDIR flow to compare.
2680  *
2681  * @return
2682  *   Zero on match, 1 otherwise.
2683  */
2684 static int
2685 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
2686 {
2687         if (FLOW_FDIR_CMP(f1, f2, attr) ||
2688             FLOW_FDIR_CMP(f1, f2, l2) ||
2689             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
2690             FLOW_FDIR_CMP(f1, f2, l3) ||
2691             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
2692             FLOW_FDIR_CMP(f1, f2, l4) ||
2693             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
2694             FLOW_FDIR_CMP(f1, f2, actions[0].type))
2695                 return 1;
2696         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
2697             FLOW_FDIR_CMP(f1, f2, queue))
2698                 return 1;
2699         return 0;
2700 }
2701
2702 /**
2703  * Search device flow list to find out a matched FDIR flow.
2704  *
2705  * @param dev
2706  *   Pointer to Ethernet device.
2707  * @param fdir_flow
2708  *   FDIR flow to lookup.
2709  *
2710  * @return
2711  *   Pointer of flow if found, NULL otherwise.
2712  */
2713 static struct rte_flow *
2714 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
2715 {
2716         struct mlx5_priv *priv = dev->data->dev_private;
2717         struct rte_flow *flow = NULL;
2718
2719         assert(fdir_flow);
2720         TAILQ_FOREACH(flow, &priv->flows, next) {
2721                 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) {
2722                         DRV_LOG(DEBUG, "port %u found FDIR flow %p",
2723                                 dev->data->port_id, (void *)flow);
2724                         break;
2725                 }
2726         }
2727         return flow;
2728 }
2729
2730 /**
2731  * Add new flow director filter and store it in list.
2732  *
2733  * @param dev
2734  *   Pointer to Ethernet device.
2735  * @param fdir_filter
2736  *   Flow director filter to add.
2737  *
2738  * @return
2739  *   0 on success, a negative errno value otherwise and rte_errno is set.
2740  */
2741 static int
2742 flow_fdir_filter_add(struct rte_eth_dev *dev,
2743                      const struct rte_eth_fdir_filter *fdir_filter)
2744 {
2745         struct mlx5_priv *priv = dev->data->dev_private;
2746         struct mlx5_fdir *fdir_flow;
2747         struct rte_flow *flow;
2748         int ret;
2749
2750         fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0);
2751         if (!fdir_flow) {
2752                 rte_errno = ENOMEM;
2753                 return -rte_errno;
2754         }
2755         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
2756         if (ret)
2757                 goto error;
2758         flow = flow_fdir_filter_lookup(dev, fdir_flow);
2759         if (flow) {
2760                 rte_errno = EEXIST;
2761                 goto error;
2762         }
2763         flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
2764                                 fdir_flow->items, fdir_flow->actions, NULL);
2765         if (!flow)
2766                 goto error;
2767         assert(!flow->fdir);
2768         flow->fdir = fdir_flow;
2769         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
2770                 dev->data->port_id, (void *)flow);
2771         return 0;
2772 error:
2773         rte_free(fdir_flow);
2774         return -rte_errno;
2775 }
2776
2777 /**
2778  * Delete specific filter.
2779  *
2780  * @param dev
2781  *   Pointer to Ethernet device.
2782  * @param fdir_filter
2783  *   Filter to be deleted.
2784  *
2785  * @return
2786  *   0 on success, a negative errno value otherwise and rte_errno is set.
2787  */
2788 static int
2789 flow_fdir_filter_delete(struct rte_eth_dev *dev,
2790                         const struct rte_eth_fdir_filter *fdir_filter)
2791 {
2792         struct mlx5_priv *priv = dev->data->dev_private;
2793         struct rte_flow *flow;
2794         struct mlx5_fdir fdir_flow = {
2795                 .attr.group = 0,
2796         };
2797         int ret;
2798
2799         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
2800         if (ret)
2801                 return -rte_errno;
2802         flow = flow_fdir_filter_lookup(dev, &fdir_flow);
2803         if (!flow) {
2804                 rte_errno = ENOENT;
2805                 return -rte_errno;
2806         }
2807         flow_list_destroy(dev, &priv->flows, flow);
2808         DRV_LOG(DEBUG, "port %u deleted FDIR flow %p",
2809                 dev->data->port_id, (void *)flow);
2810         return 0;
2811 }
2812
2813 /**
2814  * Update queue for specific filter.
2815  *
2816  * @param dev
2817  *   Pointer to Ethernet device.
2818  * @param fdir_filter
2819  *   Filter to be updated.
2820  *
2821  * @return
2822  *   0 on success, a negative errno value otherwise and rte_errno is set.
2823  */
2824 static int
2825 flow_fdir_filter_update(struct rte_eth_dev *dev,
2826                         const struct rte_eth_fdir_filter *fdir_filter)
2827 {
2828         int ret;
2829
2830         ret = flow_fdir_filter_delete(dev, fdir_filter);
2831         if (ret)
2832                 return ret;
2833         return flow_fdir_filter_add(dev, fdir_filter);
2834 }
2835
2836 /**
2837  * Flush all filters.
2838  *
2839  * @param dev
2840  *   Pointer to Ethernet device.
2841  */
2842 static void
2843 flow_fdir_filter_flush(struct rte_eth_dev *dev)
2844 {
2845         struct mlx5_priv *priv = dev->data->dev_private;
2846
2847         mlx5_flow_list_flush(dev, &priv->flows);
2848 }
2849
2850 /**
2851  * Get flow director information.
2852  *
2853  * @param dev
2854  *   Pointer to Ethernet device.
2855  * @param[out] fdir_info
2856  *   Resulting flow director information.
2857  */
2858 static void
2859 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
2860 {
2861         struct rte_eth_fdir_masks *mask =
2862                 &dev->data->dev_conf.fdir_conf.mask;
2863
2864         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
2865         fdir_info->guarant_spc = 0;
2866         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2867         fdir_info->max_flexpayload = 0;
2868         fdir_info->flow_types_mask[0] = 0;
2869         fdir_info->flex_payload_unit = 0;
2870         fdir_info->max_flex_payload_segment_num = 0;
2871         fdir_info->flex_payload_limit = 0;
2872         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2873 }
2874
2875 /**
2876  * Deal with flow director operations.
2877  *
2878  * @param dev
2879  *   Pointer to Ethernet device.
2880  * @param filter_op
2881  *   Operation to perform.
2882  * @param arg
2883  *   Pointer to operation-specific structure.
2884  *
2885  * @return
2886  *   0 on success, a negative errno value otherwise and rte_errno is set.
2887  */
2888 static int
2889 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
2890                     void *arg)
2891 {
2892         enum rte_fdir_mode fdir_mode =
2893                 dev->data->dev_conf.fdir_conf.mode;
2894
2895         if (filter_op == RTE_ETH_FILTER_NOP)
2896                 return 0;
2897         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2898             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2899                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
2900                         dev->data->port_id, fdir_mode);
2901                 rte_errno = EINVAL;
2902                 return -rte_errno;
2903         }
2904         switch (filter_op) {
2905         case RTE_ETH_FILTER_ADD:
2906                 return flow_fdir_filter_add(dev, arg);
2907         case RTE_ETH_FILTER_UPDATE:
2908                 return flow_fdir_filter_update(dev, arg);
2909         case RTE_ETH_FILTER_DELETE:
2910                 return flow_fdir_filter_delete(dev, arg);
2911         case RTE_ETH_FILTER_FLUSH:
2912                 flow_fdir_filter_flush(dev);
2913                 break;
2914         case RTE_ETH_FILTER_INFO:
2915                 flow_fdir_info_get(dev, arg);
2916                 break;
2917         default:
2918                 DRV_LOG(DEBUG, "port %u unknown operation %u",
2919                         dev->data->port_id, filter_op);
2920                 rte_errno = EINVAL;
2921                 return -rte_errno;
2922         }
2923         return 0;
2924 }
2925
2926 /**
2927  * Manage filter operations.
2928  *
2929  * @param dev
2930  *   Pointer to Ethernet device structure.
2931  * @param filter_type
2932  *   Filter type.
2933  * @param filter_op
2934  *   Operation to perform.
2935  * @param arg
2936  *   Pointer to operation-specific structure.
2937  *
2938  * @return
2939  *   0 on success, a negative errno value otherwise and rte_errno is set.
2940  */
2941 int
2942 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2943                      enum rte_filter_type filter_type,
2944                      enum rte_filter_op filter_op,
2945                      void *arg)
2946 {
2947         switch (filter_type) {
2948         case RTE_ETH_FILTER_GENERIC:
2949                 if (filter_op != RTE_ETH_FILTER_GET) {
2950                         rte_errno = EINVAL;
2951                         return -rte_errno;
2952                 }
2953                 *(const void **)arg = &mlx5_flow_ops;
2954                 return 0;
2955         case RTE_ETH_FILTER_FDIR:
2956                 return flow_fdir_ctrl_func(dev, filter_op, arg);
2957         default:
2958                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
2959                         dev->data->port_id, filter_type);
2960                 rte_errno = ENOTSUP;
2961                 return -rte_errno;
2962         }
2963         return 0;
2964 }