drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11
  12 /* Verbs header. */
  13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  14 #ifdef PEDANTIC
  15 #pragma GCC diagnostic ignored "-Wpedantic"
  16 #endif
  17 #include <infiniband/verbs.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_common.h>
  23 #include <rte_ether.h>
  24 #include <rte_eth_ctrl.h>
  25 #include <rte_ethdev_driver.h>
  26 #include <rte_flow.h>
  27 #include <rte_flow_driver.h>
  28 #include <rte_malloc.h>
  29 #include <rte_ip.h>
  30
  31 #include "mlx5.h"
  32 #include "mlx5_defs.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_glue.h"
  35 #include "mlx5_prm.h"
  36 #include "mlx5_rxtx.h"
  37
  38 /* Dev ops structure defined in mlx5.c */
  39 extern const struct eth_dev_ops mlx5_dev_ops;
  40 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
  41
  42 /** Device flow drivers. */
  43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  44 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
  45 #endif
  46 extern const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops;
  47 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  48
  49 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  50
  51 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  52         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  53 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  54         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  55 #endif
  56         [MLX5_FLOW_TYPE_TCF] = &mlx5_flow_tcf_drv_ops,
  57         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  58         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  59 };
  60
  61 enum mlx5_expansion {
  62         MLX5_EXPANSION_ROOT,
  63         MLX5_EXPANSION_ROOT_OUTER,
  64         MLX5_EXPANSION_ROOT_ETH_VLAN,
  65         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
  66         MLX5_EXPANSION_OUTER_ETH,
  67         MLX5_EXPANSION_OUTER_ETH_VLAN,
  68         MLX5_EXPANSION_OUTER_VLAN,
  69         MLX5_EXPANSION_OUTER_IPV4,
  70         MLX5_EXPANSION_OUTER_IPV4_UDP,
  71         MLX5_EXPANSION_OUTER_IPV4_TCP,
  72         MLX5_EXPANSION_OUTER_IPV6,
  73         MLX5_EXPANSION_OUTER_IPV6_UDP,
  74         MLX5_EXPANSION_OUTER_IPV6_TCP,
  75         MLX5_EXPANSION_VXLAN,
  76         MLX5_EXPANSION_VXLAN_GPE,
  77         MLX5_EXPANSION_GRE,
  78         MLX5_EXPANSION_MPLS,
  79         MLX5_EXPANSION_ETH,
  80         MLX5_EXPANSION_ETH_VLAN,
  81         MLX5_EXPANSION_VLAN,
  82         MLX5_EXPANSION_IPV4,
  83         MLX5_EXPANSION_IPV4_UDP,
  84         MLX5_EXPANSION_IPV4_TCP,
  85         MLX5_EXPANSION_IPV6,
  86         MLX5_EXPANSION_IPV6_UDP,
  87         MLX5_EXPANSION_IPV6_TCP,
  88 };
  89
  90 /** Supported expansion of items. */
  91 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
  92         [MLX5_EXPANSION_ROOT] = {
  93                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
  94                                                  MLX5_EXPANSION_IPV4,
  95                                                  MLX5_EXPANSION_IPV6),
  96                 .type = RTE_FLOW_ITEM_TYPE_END,
  97         },
  98         [MLX5_EXPANSION_ROOT_OUTER] = {
  99                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 100                                                  MLX5_EXPANSION_OUTER_IPV4,
 101                                                  MLX5_EXPANSION_OUTER_IPV6),
 102                 .type = RTE_FLOW_ITEM_TYPE_END,
 103         },
 104         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 105                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 106                 .type = RTE_FLOW_ITEM_TYPE_END,
 107         },
 108         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 109                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
 110                 .type = RTE_FLOW_ITEM_TYPE_END,
 111         },
 112         [MLX5_EXPANSION_OUTER_ETH] = {
 113                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 114                                                  MLX5_EXPANSION_OUTER_IPV6,
 115                                                  MLX5_EXPANSION_MPLS),
 116                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 117                 .rss_types = 0,
 118         },
 119         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 120                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 121                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 122                 .rss_types = 0,
 123         },
 124         [MLX5_EXPANSION_OUTER_VLAN] = {
 125                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 126                                                  MLX5_EXPANSION_OUTER_IPV6),
 127                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 128         },
 129         [MLX5_EXPANSION_OUTER_IPV4] = {
 130                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 131                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 132                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 133                          MLX5_EXPANSION_GRE),
 134                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 135                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 136                         ETH_RSS_NONFRAG_IPV4_OTHER,
 137         },
 138         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 139                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 140                                                  MLX5_EXPANSION_VXLAN_GPE),
 141                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 142                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 143         },
 144         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 145                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 146                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 147         },
 148         [MLX5_EXPANSION_OUTER_IPV6] = {
 149                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 150                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 151                          MLX5_EXPANSION_OUTER_IPV6_TCP),
 152                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 153                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 154                         ETH_RSS_NONFRAG_IPV6_OTHER,
 155         },
 156         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 157                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 158                                                  MLX5_EXPANSION_VXLAN_GPE),
 159                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 160                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 161         },
 162         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 163                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 164                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 165         },
 166         [MLX5_EXPANSION_VXLAN] = {
 167                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 168                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 169         },
 170         [MLX5_EXPANSION_VXLAN_GPE] = {
 171                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 172                                                  MLX5_EXPANSION_IPV4,
 173                                                  MLX5_EXPANSION_IPV6),
 174                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 175         },
 176         [MLX5_EXPANSION_GRE] = {
 177                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 178                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 179         },
 180         [MLX5_EXPANSION_MPLS] = {
 181                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 182                                                  MLX5_EXPANSION_IPV6),
 183                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 184         },
 185         [MLX5_EXPANSION_ETH] = {
 186                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 187                                                  MLX5_EXPANSION_IPV6),
 188                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 189         },
 190         [MLX5_EXPANSION_ETH_VLAN] = {
 191                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 192                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 193         },
 194         [MLX5_EXPANSION_VLAN] = {
 195                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 196                                                  MLX5_EXPANSION_IPV6),
 197                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 198         },
 199         [MLX5_EXPANSION_IPV4] = {
 200                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 201                                                  MLX5_EXPANSION_IPV4_TCP),
 202                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 203                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 204                         ETH_RSS_NONFRAG_IPV4_OTHER,
 205         },
 206         [MLX5_EXPANSION_IPV4_UDP] = {
 207                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 208                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 209         },
 210         [MLX5_EXPANSION_IPV4_TCP] = {
 211                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 212                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 213         },
 214         [MLX5_EXPANSION_IPV6] = {
 215                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 216                                                  MLX5_EXPANSION_IPV6_TCP),
 217                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 218                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 219                         ETH_RSS_NONFRAG_IPV6_OTHER,
 220         },
 221         [MLX5_EXPANSION_IPV6_UDP] = {
 222                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 223                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 224         },
 225         [MLX5_EXPANSION_IPV6_TCP] = {
 226                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 227                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 228         },
 229 };
 230
 231 static const struct rte_flow_ops mlx5_flow_ops = {
 232         .validate = mlx5_flow_validate,
 233         .create = mlx5_flow_create,
 234         .destroy = mlx5_flow_destroy,
 235         .flush = mlx5_flow_flush,
 236         .isolate = mlx5_flow_isolate,
 237         .query = mlx5_flow_query,
 238 };
 239
 240 /* Convert FDIR request to Generic flow. */
 241 struct mlx5_fdir {
 242         struct rte_flow_attr attr;
 243         struct rte_flow_item items[4];
 244         struct rte_flow_item_eth l2;
 245         struct rte_flow_item_eth l2_mask;
 246         union {
 247                 struct rte_flow_item_ipv4 ipv4;
 248                 struct rte_flow_item_ipv6 ipv6;
 249         } l3;
 250         union {
 251                 struct rte_flow_item_ipv4 ipv4;
 252                 struct rte_flow_item_ipv6 ipv6;
 253         } l3_mask;
 254         union {
 255                 struct rte_flow_item_udp udp;
 256                 struct rte_flow_item_tcp tcp;
 257         } l4;
 258         union {
 259                 struct rte_flow_item_udp udp;
 260                 struct rte_flow_item_tcp tcp;
 261         } l4_mask;
 262         struct rte_flow_action actions[2];
 263         struct rte_flow_action_queue queue;
 264 };
 265
 266 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
 267 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
 268         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
 269 };
 270
 271 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
 272 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 273         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
 274         { 9, 10, 11 }, { 12, 13, 14 },
 275 };
 276
 277 /* Tunnel information. */
 278 struct mlx5_flow_tunnel_info {
 279         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 280         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 281 };
 282
 283 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 284         {
 285                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 286                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 287         },
 288         {
 289                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 290                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 291         },
 292         {
 293                 .tunnel = MLX5_FLOW_LAYER_GRE,
 294                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 295         },
 296         {
 297                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 298                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 299         },
 300         {
 301                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 302                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 303         },
 304 };
 305
 306 /**
 307  * Discover the maximum number of priority available.
 308  *
 309  * @param[in] dev
 310  *   Pointer to the Ethernet device structure.
 311  *
 312  * @return
 313  *   number of supported flow priority on success, a negative errno
 314  *   value otherwise and rte_errno is set.
 315  */
 316 int
 317 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 318 {
 319         struct mlx5_priv *priv = dev->data->dev_private;
 320         struct {
 321                 struct ibv_flow_attr attr;
 322                 struct ibv_flow_spec_eth eth;
 323                 struct ibv_flow_spec_action_drop drop;
 324         } flow_attr = {
 325                 .attr = {
 326                         .num_of_specs = 2,
 327                         .port = (uint8_t)priv->ibv_port,
 328                 },
 329                 .eth = {
 330                         .type = IBV_FLOW_SPEC_ETH,
 331                         .size = sizeof(struct ibv_flow_spec_eth),
 332                 },
 333                 .drop = {
 334                         .size = sizeof(struct ibv_flow_spec_action_drop),
 335                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 336                 },
 337         };
 338         struct ibv_flow *flow;
 339         struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 340         uint16_t vprio[] = { 8, 16 };
 341         int i;
 342         int priority = 0;
 343
 344         if (!drop) {
 345                 rte_errno = ENOTSUP;
 346                 return -rte_errno;
 347         }
 348         for (i = 0; i != RTE_DIM(vprio); i++) {
 349                 flow_attr.attr.priority = vprio[i] - 1;
 350                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 351                 if (!flow)
 352                         break;
 353                 claim_zero(mlx5_glue->destroy_flow(flow));
 354                 priority = vprio[i];
 355         }
 356         mlx5_hrxq_drop_release(dev);
 357         switch (priority) {
 358         case 8:
 359                 priority = RTE_DIM(priority_map_3);
 360                 break;
 361         case 16:
 362                 priority = RTE_DIM(priority_map_5);
 363                 break;
 364         default:
 365                 rte_errno = ENOTSUP;
 366                 DRV_LOG(ERR,
 367                         "port %u verbs maximum priority: %d expected 8/16",
 368                         dev->data->port_id, priority);
 369                 return -rte_errno;
 370         }
 371         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 372                 dev->data->port_id, priority);
 373         return priority;
 374 }
 375
 376 /**
 377  * Adjust flow priority based on the highest layer and the request priority.
 378  *
 379  * @param[in] dev
 380  *   Pointer to the Ethernet device structure.
 381  * @param[in] priority
 382  *   The rule base priority.
 383  * @param[in] subpriority
 384  *   The priority based on the items.
 385  *
 386  * @return
 387  *   The new priority.
 388  */
 389 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 390                                    uint32_t subpriority)
 391 {
 392         uint32_t res = 0;
 393         struct mlx5_priv *priv = dev->data->dev_private;
 394
 395         switch (priv->config.flow_prio) {
 396         case RTE_DIM(priority_map_3):
 397                 res = priority_map_3[priority][subpriority];
 398                 break;
 399         case RTE_DIM(priority_map_5):
 400                 res = priority_map_5[priority][subpriority];
 401                 break;
 402         }
 403         return  res;
 404 }
 405
 406 /**
 407  * Verify the @p item specifications (spec, last, mask) are compatible with the
 408  * NIC capabilities.
 409  *
 410  * @param[in] item
 411  *   Item specification.
 412  * @param[in] mask
 413  *   @p item->mask or flow default bit-masks.
 414  * @param[in] nic_mask
 415  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 416  * @param[in] size
 417  *   Bit-masks size in bytes.
 418  * @param[out] error
 419  *   Pointer to error structure.
 420  *
 421  * @return
 422  *   0 on success, a negative errno value otherwise and rte_errno is set.
 423  */
 424 int
 425 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 426                           const uint8_t *mask,
 427                           const uint8_t *nic_mask,
 428                           unsigned int size,
 429                           struct rte_flow_error *error)
 430 {
 431         unsigned int i;
 432
 433         assert(nic_mask);
 434         for (i = 0; i < size; ++i)
 435                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 436                         return rte_flow_error_set(error, ENOTSUP,
 437                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 438                                                   item,
 439                                                   "mask enables non supported"
 440                                                   " bits");
 441         if (!item->spec && (item->mask || item->last))
 442                 return rte_flow_error_set(error, EINVAL,
 443                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 444                                           "mask/last without a spec is not"
 445                                           " supported");
 446         if (item->spec && item->last) {
 447                 uint8_t spec[size];
 448                 uint8_t last[size];
 449                 unsigned int i;
 450                 int ret;
 451
 452                 for (i = 0; i < size; ++i) {
 453                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 454                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 455                 }
 456                 ret = memcmp(spec, last, size);
 457                 if (ret != 0)
 458                         return rte_flow_error_set(error, EINVAL,
 459                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 460                                                   item,
 461                                                   "range is not valid");
 462         }
 463         return 0;
 464 }
 465
 466 /**
 467  * Adjust the hash fields according to the @p flow information.
 468  *
 469  * @param[in] dev_flow.
 470  *   Pointer to the mlx5_flow.
 471  * @param[in] tunnel
 472  *   1 when the hash field is for a tunnel item.
 473  * @param[in] layer_types
 474  *   ETH_RSS_* types.
 475  * @param[in] hash_fields
 476  *   Item hash fields.
 477  *
 478  * @return
 479  *   The hash fileds that should be used.
 480  */
 481 uint64_t
 482 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
 483                             int tunnel __rte_unused, uint64_t layer_types,
 484                             uint64_t hash_fields)
 485 {
 486         struct rte_flow *flow = dev_flow->flow;
 487 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 488         int rss_request_inner = flow->rss.level >= 2;
 489
 490         /* Check RSS hash level for tunnel. */
 491         if (tunnel && rss_request_inner)
 492                 hash_fields |= IBV_RX_HASH_INNER;
 493         else if (tunnel || rss_request_inner)
 494                 return 0;
 495 #endif
 496         /* Check if requested layer matches RSS hash fields. */
 497         if (!(flow->rss.types & layer_types))
 498                 return 0;
 499         return hash_fields;
 500 }
 501
 502 /**
 503  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 504  * if several tunnel rules are used on this queue, the tunnel ptype will be
 505  * cleared.
 506  *
 507  * @param rxq_ctrl
 508  *   Rx queue to update.
 509  */
 510 static void
 511 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 512 {
 513         unsigned int i;
 514         uint32_t tunnel_ptype = 0;
 515
 516         /* Look up for the ptype to use. */
 517         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 518                 if (!rxq_ctrl->flow_tunnels_n[i])
 519                         continue;
 520                 if (!tunnel_ptype) {
 521                         tunnel_ptype = tunnels_info[i].ptype;
 522                 } else {
 523                         tunnel_ptype = 0;
 524                         break;
 525                 }
 526         }
 527         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 528 }
 529
 530 /**
 531  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
 532  * flow.
 533  *
 534  * @param[in] dev
 535  *   Pointer to the Ethernet device structure.
 536  * @param[in] dev_flow
 537  *   Pointer to device flow structure.
 538  */
 539 static void
 540 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 541 {
 542         struct mlx5_priv *priv = dev->data->dev_private;
 543         struct rte_flow *flow = dev_flow->flow;
 544         const int mark = !!(flow->actions &
 545                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 546         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 547         unsigned int i;
 548
 549         for (i = 0; i != flow->rss.queue_num; ++i) {
 550                 int idx = (*flow->queue)[i];
 551                 struct mlx5_rxq_ctrl *rxq_ctrl =
 552                         container_of((*priv->rxqs)[idx],
 553                                      struct mlx5_rxq_ctrl, rxq);
 554
 555                 if (mark) {
 556                         rxq_ctrl->rxq.mark = 1;
 557                         rxq_ctrl->flow_mark_n++;
 558                 }
 559                 if (tunnel) {
 560                         unsigned int j;
 561
 562                         /* Increase the counter matching the flow. */
 563                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 564                                 if ((tunnels_info[j].tunnel &
 565                                      dev_flow->layers) ==
 566                                     tunnels_info[j].tunnel) {
 567                                         rxq_ctrl->flow_tunnels_n[j]++;
 568                                         break;
 569                                 }
 570                         }
 571                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 572                 }
 573         }
 574 }
 575
 576 /**
 577  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
 578  *
 579  * @param[in] dev
 580  *   Pointer to the Ethernet device structure.
 581  * @param[in] flow
 582  *   Pointer to flow structure.
 583  */
 584 static void
 585 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 586 {
 587         struct mlx5_flow *dev_flow;
 588
 589         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 590                 flow_drv_rxq_flags_set(dev, dev_flow);
 591 }
 592
 593 /**
 594  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 595  * device flow if no other flow uses it with the same kind of request.
 596  *
 597  * @param dev
 598  *   Pointer to Ethernet device.
 599  * @param[in] dev_flow
 600  *   Pointer to the device flow.
 601  */
 602 static void
 603 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
 604 {
 605         struct mlx5_priv *priv = dev->data->dev_private;
 606         struct rte_flow *flow = dev_flow->flow;
 607         const int mark = !!(flow->actions &
 608                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 609         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 610         unsigned int i;
 611
 612         assert(dev->data->dev_started);
 613         for (i = 0; i != flow->rss.queue_num; ++i) {
 614                 int idx = (*flow->queue)[i];
 615                 struct mlx5_rxq_ctrl *rxq_ctrl =
 616                         container_of((*priv->rxqs)[idx],
 617                                      struct mlx5_rxq_ctrl, rxq);
 618
 619                 if (mark) {
 620                         rxq_ctrl->flow_mark_n--;
 621                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 622                 }
 623                 if (tunnel) {
 624                         unsigned int j;
 625
 626                         /* Decrease the counter matching the flow. */
 627                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 628                                 if ((tunnels_info[j].tunnel &
 629                                      dev_flow->layers) ==
 630                                     tunnels_info[j].tunnel) {
 631                                         rxq_ctrl->flow_tunnels_n[j]--;
 632                                         break;
 633                                 }
 634                         }
 635                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
 636                 }
 637         }
 638 }
 639
 640 /**
 641  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 642  * @p flow if no other flow uses it with the same kind of request.
 643  *
 644  * @param dev
 645  *   Pointer to Ethernet device.
 646  * @param[in] flow
 647  *   Pointer to the flow.
 648  */
 649 static void
 650 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 651 {
 652         struct mlx5_flow *dev_flow;
 653
 654         LIST_FOREACH(dev_flow, &flow->dev_flows, next)
 655                 flow_drv_rxq_flags_trim(dev, dev_flow);
 656 }
 657
 658 /**
 659  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
 660  *
 661  * @param dev
 662  *   Pointer to Ethernet device.
 663  */
 664 static void
 665 flow_rxq_flags_clear(struct rte_eth_dev *dev)
 666 {
 667         struct mlx5_priv *priv = dev->data->dev_private;
 668         unsigned int i;
 669
 670         for (i = 0; i != priv->rxqs_n; ++i) {
 671                 struct mlx5_rxq_ctrl *rxq_ctrl;
 672                 unsigned int j;
 673
 674                 if (!(*priv->rxqs)[i])
 675                         continue;
 676                 rxq_ctrl = container_of((*priv->rxqs)[i],
 677                                         struct mlx5_rxq_ctrl, rxq);
 678                 rxq_ctrl->flow_mark_n = 0;
 679                 rxq_ctrl->rxq.mark = 0;
 680                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
 681                         rxq_ctrl->flow_tunnels_n[j] = 0;
 682                 rxq_ctrl->rxq.tunnel = 0;
 683         }
 684 }
 685
 686 /*
 687  * Validate the flag action.
 688  *
 689  * @param[in] action_flags
 690  *   Bit-fields that holds the actions detected until now.
 691  * @param[in] attr
 692  *   Attributes of flow that includes this action.
 693  * @param[out] error
 694  *   Pointer to error structure.
 695  *
 696  * @return
 697  *   0 on success, a negative errno value otherwise and rte_errno is set.
 698  */
 699 int
 700 mlx5_flow_validate_action_flag(uint64_t action_flags,
 701                                const struct rte_flow_attr *attr,
 702                                struct rte_flow_error *error)
 703 {
 704
 705         if (action_flags & MLX5_FLOW_ACTION_DROP)
 706                 return rte_flow_error_set(error, EINVAL,
 707                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 708                                           "can't drop and flag in same flow");
 709         if (action_flags & MLX5_FLOW_ACTION_MARK)
 710                 return rte_flow_error_set(error, EINVAL,
 711                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 712                                           "can't mark and flag in same flow");
 713         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 714                 return rte_flow_error_set(error, EINVAL,
 715                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 716                                           "can't have 2 flag"
 717                                           " actions in same flow");
 718         if (attr->egress)
 719                 return rte_flow_error_set(error, ENOTSUP,
 720                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 721                                           "flag action not supported for "
 722                                           "egress");
 723         return 0;
 724 }
 725
 726 /*
 727  * Validate the mark action.
 728  *
 729  * @param[in] action
 730  *   Pointer to the queue action.
 731  * @param[in] action_flags
 732  *   Bit-fields that holds the actions detected until now.
 733  * @param[in] attr
 734  *   Attributes of flow that includes this action.
 735  * @param[out] error
 736  *   Pointer to error structure.
 737  *
 738  * @return
 739  *   0 on success, a negative errno value otherwise and rte_errno is set.
 740  */
 741 int
 742 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
 743                                uint64_t action_flags,
 744                                const struct rte_flow_attr *attr,
 745                                struct rte_flow_error *error)
 746 {
 747         const struct rte_flow_action_mark *mark = action->conf;
 748
 749         if (!mark)
 750                 return rte_flow_error_set(error, EINVAL,
 751                                           RTE_FLOW_ERROR_TYPE_ACTION,
 752                                           action,
 753                                           "configuration cannot be null");
 754         if (mark->id >= MLX5_FLOW_MARK_MAX)
 755                 return rte_flow_error_set(error, EINVAL,
 756                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 757                                           &mark->id,
 758                                           "mark id must in 0 <= id < "
 759                                           RTE_STR(MLX5_FLOW_MARK_MAX));
 760         if (action_flags & MLX5_FLOW_ACTION_DROP)
 761                 return rte_flow_error_set(error, EINVAL,
 762                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 763                                           "can't drop and mark in same flow");
 764         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 765                 return rte_flow_error_set(error, EINVAL,
 766                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 767                                           "can't flag and mark in same flow");
 768         if (action_flags & MLX5_FLOW_ACTION_MARK)
 769                 return rte_flow_error_set(error, EINVAL,
 770                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 771                                           "can't have 2 mark actions in same"
 772                                           " flow");
 773         if (attr->egress)
 774                 return rte_flow_error_set(error, ENOTSUP,
 775                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 776                                           "mark action not supported for "
 777                                           "egress");
 778         return 0;
 779 }
 780
 781 /*
 782  * Validate the drop action.
 783  *
 784  * @param[in] action_flags
 785  *   Bit-fields that holds the actions detected until now.
 786  * @param[in] attr
 787  *   Attributes of flow that includes this action.
 788  * @param[out] error
 789  *   Pointer to error structure.
 790  *
 791  * @return
 792  *   0 on success, a negative errno value otherwise and rte_errno is set.
 793  */
 794 int
 795 mlx5_flow_validate_action_drop(uint64_t action_flags,
 796                                const struct rte_flow_attr *attr,
 797                                struct rte_flow_error *error)
 798 {
 799         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 800                 return rte_flow_error_set(error, EINVAL,
 801                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 802                                           "can't drop and flag in same flow");
 803         if (action_flags & MLX5_FLOW_ACTION_MARK)
 804                 return rte_flow_error_set(error, EINVAL,
 805                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 806                                           "can't drop and mark in same flow");
 807         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 808                 return rte_flow_error_set(error, EINVAL,
 809                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 810                                           "can't have 2 fate actions in"
 811                                           " same flow");
 812         if (attr->egress)
 813                 return rte_flow_error_set(error, ENOTSUP,
 814                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 815                                           "drop action not supported for "
 816                                           "egress");
 817         return 0;
 818 }
 819
 820 /*
 821  * Validate the queue action.
 822  *
 823  * @param[in] action
 824  *   Pointer to the queue action.
 825  * @param[in] action_flags
 826  *   Bit-fields that holds the actions detected until now.
 827  * @param[in] dev
 828  *   Pointer to the Ethernet device structure.
 829  * @param[in] attr
 830  *   Attributes of flow that includes this action.
 831  * @param[out] error
 832  *   Pointer to error structure.
 833  *
 834  * @return
 835  *   0 on success, a negative errno value otherwise and rte_errno is set.
 836  */
 837 int
 838 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
 839                                 uint64_t action_flags,
 840                                 struct rte_eth_dev *dev,
 841                                 const struct rte_flow_attr *attr,
 842                                 struct rte_flow_error *error)
 843 {
 844         struct mlx5_priv *priv = dev->data->dev_private;
 845         const struct rte_flow_action_queue *queue = action->conf;
 846
 847         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 848                 return rte_flow_error_set(error, EINVAL,
 849                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 850                                           "can't have 2 fate actions in"
 851                                           " same flow");
 852         if (!priv->rxqs_n)
 853                 return rte_flow_error_set(error, EINVAL,
 854                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 855                                           NULL, "No Rx queues configured");
 856         if (queue->index >= priv->rxqs_n)
 857                 return rte_flow_error_set(error, EINVAL,
 858                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 859                                           &queue->index,
 860                                           "queue index out of range");
 861         if (!(*priv->rxqs)[queue->index])
 862                 return rte_flow_error_set(error, EINVAL,
 863                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 864                                           &queue->index,
 865                                           "queue is not configured");
 866         if (attr->egress)
 867                 return rte_flow_error_set(error, ENOTSUP,
 868                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 869                                           "queue action not supported for "
 870                                           "egress");
 871         return 0;
 872 }
 873
 874 /*
 875  * Validate the rss action.
 876  *
 877  * @param[in] action
 878  *   Pointer to the queue action.
 879  * @param[in] action_flags
 880  *   Bit-fields that holds the actions detected until now.
 881  * @param[in] dev
 882  *   Pointer to the Ethernet device structure.
 883  * @param[in] attr
 884  *   Attributes of flow that includes this action.
 885  * @param[in] item_flags
 886  *   Items that were detected.
 887  * @param[out] error
 888  *   Pointer to error structure.
 889  *
 890  * @return
 891  *   0 on success, a negative errno value otherwise and rte_errno is set.
 892  */
 893 int
 894 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
 895                               uint64_t action_flags,
 896                               struct rte_eth_dev *dev,
 897                               const struct rte_flow_attr *attr,
 898                               uint64_t item_flags,
 899                               struct rte_flow_error *error)
 900 {
 901         struct mlx5_priv *priv = dev->data->dev_private;
 902         const struct rte_flow_action_rss *rss = action->conf;
 903         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 904         unsigned int i;
 905
 906         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 907                 return rte_flow_error_set(error, EINVAL,
 908                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 909                                           "can't have 2 fate actions"
 910                                           " in same flow");
 911         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
 912             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 913                 return rte_flow_error_set(error, ENOTSUP,
 914                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 915                                           &rss->func,
 916                                           "RSS hash function not supported");
 917 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 918         if (rss->level > 2)
 919 #else
 920         if (rss->level > 1)
 921 #endif
 922                 return rte_flow_error_set(error, ENOTSUP,
 923                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 924                                           &rss->level,
 925                                           "tunnel RSS is not supported");
 926         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
 927         if (rss->key_len == 0 && rss->key != NULL)
 928                 return rte_flow_error_set(error, ENOTSUP,
 929                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 930                                           &rss->key_len,
 931                                           "RSS hash key length 0");
 932         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
 933                 return rte_flow_error_set(error, ENOTSUP,
 934                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 935                                           &rss->key_len,
 936                                           "RSS hash key too small");
 937         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
 938                 return rte_flow_error_set(error, ENOTSUP,
 939                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 940                                           &rss->key_len,
 941                                           "RSS hash key too large");
 942         if (rss->queue_num > priv->config.ind_table_max_size)
 943                 return rte_flow_error_set(error, ENOTSUP,
 944                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 945                                           &rss->queue_num,
 946                                           "number of queues too large");
 947         if (rss->types & MLX5_RSS_HF_MASK)
 948                 return rte_flow_error_set(error, ENOTSUP,
 949                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 950                                           &rss->types,
 951                                           "some RSS protocols are not"
 952                                           " supported");
 953         if (!priv->rxqs_n)
 954                 return rte_flow_error_set(error, EINVAL,
 955                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 956                                           NULL, "No Rx queues configured");
 957         if (!rss->queue_num)
 958                 return rte_flow_error_set(error, EINVAL,
 959                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 960                                           NULL, "No queues configured");
 961         for (i = 0; i != rss->queue_num; ++i) {
 962                 if (!(*priv->rxqs)[rss->queue[i]])
 963                         return rte_flow_error_set
 964                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 965                                  &rss->queue[i], "queue is not configured");
 966         }
 967         if (attr->egress)
 968                 return rte_flow_error_set(error, ENOTSUP,
 969                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 970                                           "rss action not supported for "
 971                                           "egress");
 972         if (rss->level > 1 &&  !tunnel)
 973                 return rte_flow_error_set(error, EINVAL,
 974                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
 975                                           "inner RSS is not supported for "
 976                                           "non-tunnel flows");
 977         return 0;
 978 }
 979
 980 /*
 981  * Validate the count action.
 982  *
 983  * @param[in] dev
 984  *   Pointer to the Ethernet device structure.
 985  * @param[in] attr
 986  *   Attributes of flow that includes this action.
 987  * @param[out] error
 988  *   Pointer to error structure.
 989  *
 990  * @return
 991  *   0 on success, a negative errno value otherwise and rte_errno is set.
 992  */
 993 int
 994 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
 995                                 const struct rte_flow_attr *attr,
 996                                 struct rte_flow_error *error)
 997 {
 998         if (attr->egress)
 999                 return rte_flow_error_set(error, ENOTSUP,
1000                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1001                                           "count action not supported for "
1002                                           "egress");
1003         return 0;
1004 }
1005
1006 /**
1007  * Verify the @p attributes will be correctly understood by the NIC and store
1008  * them in the @p flow if everything is correct.
1009  *
1010  * @param[in] dev
1011  *   Pointer to the Ethernet device structure.
1012  * @param[in] attributes
1013  *   Pointer to flow attributes
1014  * @param[out] error
1015  *   Pointer to error structure.
1016  *
1017  * @return
1018  *   0 on success, a negative errno value otherwise and rte_errno is set.
1019  */
1020 int
1021 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1022                               const struct rte_flow_attr *attributes,
1023                               struct rte_flow_error *error)
1024 {
1025         struct mlx5_priv *priv = dev->data->dev_private;
1026         uint32_t priority_max = priv->config.flow_prio - 1;
1027
1028         if (attributes->group)
1029                 return rte_flow_error_set(error, ENOTSUP,
1030                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1031                                           NULL, "groups is not supported");
1032         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1033             attributes->priority >= priority_max)
1034                 return rte_flow_error_set(error, ENOTSUP,
1035                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1036                                           NULL, "priority out of range");
1037         if (attributes->egress)
1038                 return rte_flow_error_set(error, ENOTSUP,
1039                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1040                                           "egress is not supported");
1041         if (attributes->transfer)
1042                 return rte_flow_error_set(error, ENOTSUP,
1043                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1044                                           NULL, "transfer is not supported");
1045         if (!attributes->ingress)
1046                 return rte_flow_error_set(error, EINVAL,
1047                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1048                                           NULL,
1049                                           "ingress attribute is mandatory");
1050         return 0;
1051 }
1052
1053 /**
1054  * Validate Ethernet item.
1055  *
1056  * @param[in] item
1057  *   Item specification.
1058  * @param[in] item_flags
1059  *   Bit-fields that holds the items detected until now.
1060  * @param[out] error
1061  *   Pointer to error structure.
1062  *
1063  * @return
1064  *   0 on success, a negative errno value otherwise and rte_errno is set.
1065  */
1066 int
1067 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1068                             uint64_t item_flags,
1069                             struct rte_flow_error *error)
1070 {
1071         const struct rte_flow_item_eth *mask = item->mask;
1072         const struct rte_flow_item_eth nic_mask = {
1073                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1074                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1075                 .type = RTE_BE16(0xffff),
1076         };
1077         int ret;
1078         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1079         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1080                                        MLX5_FLOW_LAYER_OUTER_L2;
1081
1082         if (item_flags & ethm)
1083                 return rte_flow_error_set(error, ENOTSUP,
1084                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1085                                           "multiple L2 layers not supported");
1086         if (!mask)
1087                 mask = &rte_flow_item_eth_mask;
1088         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1089                                         (const uint8_t *)&nic_mask,
1090                                         sizeof(struct rte_flow_item_eth),
1091                                         error);
1092         return ret;
1093 }
1094
1095 /**
1096  * Validate VLAN item.
1097  *
1098  * @param[in] item
1099  *   Item specification.
1100  * @param[in] item_flags
1101  *   Bit-fields that holds the items detected until now.
1102  * @param[out] error
1103  *   Pointer to error structure.
1104  *
1105  * @return
1106  *   0 on success, a negative errno value otherwise and rte_errno is set.
1107  */
1108 int
1109 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1110                              uint64_t item_flags,
1111                              struct rte_flow_error *error)
1112 {
1113         const struct rte_flow_item_vlan *spec = item->spec;
1114         const struct rte_flow_item_vlan *mask = item->mask;
1115         const struct rte_flow_item_vlan nic_mask = {
1116                 .tci = RTE_BE16(0x0fff),
1117                 .inner_type = RTE_BE16(0xffff),
1118         };
1119         uint16_t vlan_tag = 0;
1120         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1121         int ret;
1122         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1123                                         MLX5_FLOW_LAYER_INNER_L4) :
1124                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1125                                         MLX5_FLOW_LAYER_OUTER_L4);
1126         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1127                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1128
1129         if (item_flags & vlanm)
1130                 return rte_flow_error_set(error, EINVAL,
1131                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1132                                           "multiple VLAN layers not supported");
1133         else if ((item_flags & l34m) != 0)
1134                 return rte_flow_error_set(error, EINVAL,
1135                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1136                                           "L2 layer cannot follow L3/L4 layer");
1137         if (!mask)
1138                 mask = &rte_flow_item_vlan_mask;
1139         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1140                                         (const uint8_t *)&nic_mask,
1141                                         sizeof(struct rte_flow_item_vlan),
1142                                         error);
1143         if (ret)
1144                 return ret;
1145         if (spec) {
1146                 vlan_tag = spec->tci;
1147                 vlan_tag &= mask->tci;
1148         }
1149         /*
1150          * From verbs perspective an empty VLAN is equivalent
1151          * to a packet without VLAN layer.
1152          */
1153         if (!vlan_tag)
1154                 return rte_flow_error_set(error, EINVAL,
1155                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1156                                           item->spec,
1157                                           "VLAN cannot be empty");
1158         return 0;
1159 }
1160
1161 /**
1162  * Validate IPV4 item.
1163  *
1164  * @param[in] item
1165  *   Item specification.
1166  * @param[in] item_flags
1167  *   Bit-fields that holds the items detected until now.
1168  * @param[in] acc_mask
1169  *   Acceptable mask, if NULL default internal default mask
1170  *   will be used to check whether item fields are supported.
1171  * @param[out] error
1172  *   Pointer to error structure.
1173  *
1174  * @return
1175  *   0 on success, a negative errno value otherwise and rte_errno is set.
1176  */
1177 int
1178 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1179                              uint64_t item_flags,
1180                              const struct rte_flow_item_ipv4 *acc_mask,
1181                              struct rte_flow_error *error)
1182 {
1183         const struct rte_flow_item_ipv4 *mask = item->mask;
1184         const struct rte_flow_item_ipv4 nic_mask = {
1185                 .hdr = {
1186                         .src_addr = RTE_BE32(0xffffffff),
1187                         .dst_addr = RTE_BE32(0xffffffff),
1188                         .type_of_service = 0xff,
1189                         .next_proto_id = 0xff,
1190                 },
1191         };
1192         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1193         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1194                                       MLX5_FLOW_LAYER_OUTER_L3;
1195         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1196                                       MLX5_FLOW_LAYER_OUTER_L4;
1197         int ret;
1198
1199         if (item_flags & l3m)
1200                 return rte_flow_error_set(error, ENOTSUP,
1201                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1202                                           "multiple L3 layers not supported");
1203         else if (item_flags & l4m)
1204                 return rte_flow_error_set(error, EINVAL,
1205                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1206                                           "L3 cannot follow an L4 layer.");
1207         if (!mask)
1208                 mask = &rte_flow_item_ipv4_mask;
1209         else if (mask->hdr.next_proto_id != 0 &&
1210                  mask->hdr.next_proto_id != 0xff)
1211                 return rte_flow_error_set(error, EINVAL,
1212                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1213                                           "partial mask is not supported"
1214                                           " for protocol");
1215         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1216                                         acc_mask ? (const uint8_t *)acc_mask
1217                                                  : (const uint8_t *)&nic_mask,
1218                                         sizeof(struct rte_flow_item_ipv4),
1219                                         error);
1220         if (ret < 0)
1221                 return ret;
1222         return 0;
1223 }
1224
1225 /**
1226  * Validate IPV6 item.
1227  *
1228  * @param[in] item
1229  *   Item specification.
1230  * @param[in] item_flags
1231  *   Bit-fields that holds the items detected until now.
1232  * @param[in] acc_mask
1233  *   Acceptable mask, if NULL default internal default mask
1234  *   will be used to check whether item fields are supported.
1235  * @param[out] error
1236  *   Pointer to error structure.
1237  *
1238  * @return
1239  *   0 on success, a negative errno value otherwise and rte_errno is set.
1240  */
1241 int
1242 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1243                              uint64_t item_flags,
1244                              const struct rte_flow_item_ipv6 *acc_mask,
1245                              struct rte_flow_error *error)
1246 {
1247         const struct rte_flow_item_ipv6 *mask = item->mask;
1248         const struct rte_flow_item_ipv6 nic_mask = {
1249                 .hdr = {
1250                         .src_addr =
1251                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1252                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1253                         .dst_addr =
1254                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1255                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1256                         .vtc_flow = RTE_BE32(0xffffffff),
1257                         .proto = 0xff,
1258                         .hop_limits = 0xff,
1259                 },
1260         };
1261         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1262         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1263                                       MLX5_FLOW_LAYER_OUTER_L3;
1264         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1265                                       MLX5_FLOW_LAYER_OUTER_L4;
1266         int ret;
1267
1268         if (item_flags & l3m)
1269                 return rte_flow_error_set(error, ENOTSUP,
1270                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1271                                           "multiple L3 layers not supported");
1272         else if (item_flags & l4m)
1273                 return rte_flow_error_set(error, EINVAL,
1274                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1275                                           "L3 cannot follow an L4 layer.");
1276         if (!mask)
1277                 mask = &rte_flow_item_ipv6_mask;
1278         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1279                                         acc_mask ? (const uint8_t *)acc_mask
1280                                                  : (const uint8_t *)&nic_mask,
1281                                         sizeof(struct rte_flow_item_ipv6),
1282                                         error);
1283         if (ret < 0)
1284                 return ret;
1285         return 0;
1286 }
1287
1288 /**
1289  * Validate UDP item.
1290  *
1291  * @param[in] item
1292  *   Item specification.
1293  * @param[in] item_flags
1294  *   Bit-fields that holds the items detected until now.
1295  * @param[in] target_protocol
1296  *   The next protocol in the previous item.
1297  * @param[in] flow_mask
1298  *   mlx5 flow-specific (TCF, DV, verbs, etc.) supported header fields mask.
1299  * @param[out] error
1300  *   Pointer to error structure.
1301  *
1302  * @return
1303  *   0 on success, a negative errno value otherwise and rte_errno is set.
1304  */
1305 int
1306 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1307                             uint64_t item_flags,
1308                             uint8_t target_protocol,
1309                             struct rte_flow_error *error)
1310 {
1311         const struct rte_flow_item_udp *mask = item->mask;
1312         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1313         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1314                                       MLX5_FLOW_LAYER_OUTER_L3;
1315         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1316                                       MLX5_FLOW_LAYER_OUTER_L4;
1317         int ret;
1318
1319         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1320                 return rte_flow_error_set(error, EINVAL,
1321                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1322                                           "protocol filtering not compatible"
1323                                           " with UDP layer");
1324         if (!(item_flags & l3m))
1325                 return rte_flow_error_set(error, EINVAL,
1326                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1327                                           "L3 is mandatory to filter on L4");
1328         if (item_flags & l4m)
1329                 return rte_flow_error_set(error, EINVAL,
1330                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1331                                           "multiple L4 layers not supported");
1332         if (!mask)
1333                 mask = &rte_flow_item_udp_mask;
1334         ret = mlx5_flow_item_acceptable
1335                 (item, (const uint8_t *)mask,
1336                  (const uint8_t *)&rte_flow_item_udp_mask,
1337                  sizeof(struct rte_flow_item_udp), error);
1338         if (ret < 0)
1339                 return ret;
1340         return 0;
1341 }
1342
1343 /**
1344  * Validate TCP item.
1345  *
1346  * @param[in] item
1347  *   Item specification.
1348  * @param[in] item_flags
1349  *   Bit-fields that holds the items detected until now.
1350  * @param[in] target_protocol
1351  *   The next protocol in the previous item.
1352  * @param[out] error
1353  *   Pointer to error structure.
1354  *
1355  * @return
1356  *   0 on success, a negative errno value otherwise and rte_errno is set.
1357  */
1358 int
1359 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1360                             uint64_t item_flags,
1361                             uint8_t target_protocol,
1362                             const struct rte_flow_item_tcp *flow_mask,
1363                             struct rte_flow_error *error)
1364 {
1365         const struct rte_flow_item_tcp *mask = item->mask;
1366         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1367         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1368                                       MLX5_FLOW_LAYER_OUTER_L3;
1369         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1370                                       MLX5_FLOW_LAYER_OUTER_L4;
1371         int ret;
1372
1373         assert(flow_mask);
1374         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1375                 return rte_flow_error_set(error, EINVAL,
1376                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1377                                           "protocol filtering not compatible"
1378                                           " with TCP layer");
1379         if (!(item_flags & l3m))
1380                 return rte_flow_error_set(error, EINVAL,
1381                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1382                                           "L3 is mandatory to filter on L4");
1383         if (item_flags & l4m)
1384                 return rte_flow_error_set(error, EINVAL,
1385                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1386                                           "multiple L4 layers not supported");
1387         if (!mask)
1388                 mask = &rte_flow_item_tcp_mask;
1389         ret = mlx5_flow_item_acceptable
1390                 (item, (const uint8_t *)mask,
1391                  (const uint8_t *)flow_mask,
1392                  sizeof(struct rte_flow_item_tcp), error);
1393         if (ret < 0)
1394                 return ret;
1395         return 0;
1396 }
1397
1398 /**
1399  * Validate VXLAN item.
1400  *
1401  * @param[in] item
1402  *   Item specification.
1403  * @param[in] item_flags
1404  *   Bit-fields that holds the items detected until now.
1405  * @param[in] target_protocol
1406  *   The next protocol in the previous item.
1407  * @param[out] error
1408  *   Pointer to error structure.
1409  *
1410  * @return
1411  *   0 on success, a negative errno value otherwise and rte_errno is set.
1412  */
1413 int
1414 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1415                               uint64_t item_flags,
1416                               struct rte_flow_error *error)
1417 {
1418         const struct rte_flow_item_vxlan *spec = item->spec;
1419         const struct rte_flow_item_vxlan *mask = item->mask;
1420         int ret;
1421         union vni {
1422                 uint32_t vlan_id;
1423                 uint8_t vni[4];
1424         } id = { .vlan_id = 0, };
1425         uint32_t vlan_id = 0;
1426
1427
1428         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1429                 return rte_flow_error_set(error, ENOTSUP,
1430                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1431                                           "multiple tunnel layers not"
1432                                           " supported");
1433         /*
1434          * Verify only UDPv4 is present as defined in
1435          * https://tools.ietf.org/html/rfc7348
1436          */
1437         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1438                 return rte_flow_error_set(error, EINVAL,
1439                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1440                                           "no outer UDP layer found");
1441         if (!mask)
1442                 mask = &rte_flow_item_vxlan_mask;
1443         ret = mlx5_flow_item_acceptable
1444                 (item, (const uint8_t *)mask,
1445                  (const uint8_t *)&rte_flow_item_vxlan_mask,
1446                  sizeof(struct rte_flow_item_vxlan),
1447                  error);
1448         if (ret < 0)
1449                 return ret;
1450         if (spec) {
1451                 memcpy(&id.vni[1], spec->vni, 3);
1452                 vlan_id = id.vlan_id;
1453                 memcpy(&id.vni[1], mask->vni, 3);
1454                 vlan_id &= id.vlan_id;
1455         }
1456         /*
1457          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
1458          * only this layer is defined in the Verbs specification it is
1459          * interpreted as wildcard and all packets will match this
1460          * rule, if it follows a full stack layer (ex: eth / ipv4 /
1461          * udp), all packets matching the layers before will also
1462          * match this rule.  To avoid such situation, VNI 0 is
1463          * currently refused.
1464          */
1465         if (!vlan_id)
1466                 return rte_flow_error_set(error, ENOTSUP,
1467                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1468                                           "VXLAN vni cannot be 0");
1469         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1470                 return rte_flow_error_set(error, ENOTSUP,
1471                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1472                                           "VXLAN tunnel must be fully defined");
1473         return 0;
1474 }
1475
1476 /**
1477  * Validate VXLAN_GPE item.
1478  *
1479  * @param[in] item
1480  *   Item specification.
1481  * @param[in] item_flags
1482  *   Bit-fields that holds the items detected until now.
1483  * @param[in] priv
1484  *   Pointer to the private data structure.
1485  * @param[in] target_protocol
1486  *   The next protocol in the previous item.
1487  * @param[out] error
1488  *   Pointer to error structure.
1489  *
1490  * @return
1491  *   0 on success, a negative errno value otherwise and rte_errno is set.
1492  */
1493 int
1494 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1495                                   uint64_t item_flags,
1496                                   struct rte_eth_dev *dev,
1497                                   struct rte_flow_error *error)
1498 {
1499         struct mlx5_priv *priv = dev->data->dev_private;
1500         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1501         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1502         int ret;
1503         union vni {
1504                 uint32_t vlan_id;
1505                 uint8_t vni[4];
1506         } id = { .vlan_id = 0, };
1507         uint32_t vlan_id = 0;
1508
1509         if (!priv->config.l3_vxlan_en)
1510                 return rte_flow_error_set(error, ENOTSUP,
1511                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1512                                           "L3 VXLAN is not enabled by device"
1513                                           " parameter and/or not configured in"
1514                                           " firmware");
1515         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1516                 return rte_flow_error_set(error, ENOTSUP,
1517                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1518                                           "multiple tunnel layers not"
1519                                           " supported");
1520         /*
1521          * Verify only UDPv4 is present as defined in
1522          * https://tools.ietf.org/html/rfc7348
1523          */
1524         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1525                 return rte_flow_error_set(error, EINVAL,
1526                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1527                                           "no outer UDP layer found");
1528         if (!mask)
1529                 mask = &rte_flow_item_vxlan_gpe_mask;
1530         ret = mlx5_flow_item_acceptable
1531                 (item, (const uint8_t *)mask,
1532                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1533                  sizeof(struct rte_flow_item_vxlan_gpe),
1534                  error);
1535         if (ret < 0)
1536                 return ret;
1537         if (spec) {
1538                 if (spec->protocol)
1539                         return rte_flow_error_set(error, ENOTSUP,
1540                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1541                                                   item,
1542                                                   "VxLAN-GPE protocol"
1543                                                   " not supported");
1544                 memcpy(&id.vni[1], spec->vni, 3);
1545                 vlan_id = id.vlan_id;
1546                 memcpy(&id.vni[1], mask->vni, 3);
1547                 vlan_id &= id.vlan_id;
1548         }
1549         /*
1550          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1551          * layer is defined in the Verbs specification it is interpreted as
1552          * wildcard and all packets will match this rule, if it follows a full
1553          * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1554          * before will also match this rule.  To avoid such situation, VNI 0
1555          * is currently refused.
1556          */
1557         if (!vlan_id)
1558                 return rte_flow_error_set(error, ENOTSUP,
1559                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1560                                           "VXLAN-GPE vni cannot be 0");
1561         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1562                 return rte_flow_error_set(error, ENOTSUP,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1564                                           "VXLAN-GPE tunnel must be fully"
1565                                           " defined");
1566         return 0;
1567 }
1568
1569 /**
1570  * Validate GRE item.
1571  *
1572  * @param[in] item
1573  *   Item specification.
1574  * @param[in] item_flags
1575  *   Bit flags to mark detected items.
1576  * @param[in] target_protocol
1577  *   The next protocol in the previous item.
1578  * @param[out] error
1579  *   Pointer to error structure.
1580  *
1581  * @return
1582  *   0 on success, a negative errno value otherwise and rte_errno is set.
1583  */
1584 int
1585 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
1586                             uint64_t item_flags,
1587                             uint8_t target_protocol,
1588                             struct rte_flow_error *error)
1589 {
1590         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
1591         const struct rte_flow_item_gre *mask = item->mask;
1592         int ret;
1593
1594         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
1595                 return rte_flow_error_set(error, EINVAL,
1596                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1597                                           "protocol filtering not compatible"
1598                                           " with this GRE layer");
1599         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1600                 return rte_flow_error_set(error, ENOTSUP,
1601                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1602                                           "multiple tunnel layers not"
1603                                           " supported");
1604         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1605                 return rte_flow_error_set(error, ENOTSUP,
1606                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1607                                           "L3 Layer is missing");
1608         if (!mask)
1609                 mask = &rte_flow_item_gre_mask;
1610         ret = mlx5_flow_item_acceptable
1611                 (item, (const uint8_t *)mask,
1612                  (const uint8_t *)&rte_flow_item_gre_mask,
1613                  sizeof(struct rte_flow_item_gre), error);
1614         if (ret < 0)
1615                 return ret;
1616 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
1617         if (spec && (spec->protocol & mask->protocol))
1618                 return rte_flow_error_set(error, ENOTSUP,
1619                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1620                                           "without MPLS support the"
1621                                           " specification cannot be used for"
1622                                           " filtering");
1623 #endif
1624         return 0;
1625 }
1626
1627 /**
1628  * Validate MPLS item.
1629  *
1630  * @param[in] dev
1631  *   Pointer to the rte_eth_dev structure.
1632  * @param[in] item
1633  *   Item specification.
1634  * @param[in] item_flags
1635  *   Bit-fields that holds the items detected until now.
1636  * @param[in] prev_layer
1637  *   The protocol layer indicated in previous item.
1638  * @param[out] error
1639  *   Pointer to error structure.
1640  *
1641  * @return
1642  *   0 on success, a negative errno value otherwise and rte_errno is set.
1643  */
1644 int
1645 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
1646                              const struct rte_flow_item *item __rte_unused,
1647                              uint64_t item_flags __rte_unused,
1648                              uint64_t prev_layer __rte_unused,
1649                              struct rte_flow_error *error)
1650 {
1651 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1652         const struct rte_flow_item_mpls *mask = item->mask;
1653         struct mlx5_priv *priv = dev->data->dev_private;
1654         int ret;
1655
1656         if (!priv->config.mpls_en)
1657                 return rte_flow_error_set(error, ENOTSUP,
1658                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1659                                           "MPLS not supported or"
1660                                           " disabled in firmware"
1661                                           " configuration.");
1662         /* MPLS over IP, UDP, GRE is allowed */
1663         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
1664                             MLX5_FLOW_LAYER_OUTER_L4_UDP |
1665                             MLX5_FLOW_LAYER_GRE)))
1666                 return rte_flow_error_set(error, EINVAL,
1667                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1668                                           "protocol filtering not compatible"
1669                                           " with MPLS layer");
1670         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
1671         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1672             !(item_flags & MLX5_FLOW_LAYER_GRE))
1673                 return rte_flow_error_set(error, ENOTSUP,
1674                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1675                                           "multiple tunnel layers not"
1676                                           " supported");
1677         if (!mask)
1678                 mask = &rte_flow_item_mpls_mask;
1679         ret = mlx5_flow_item_acceptable
1680                 (item, (const uint8_t *)mask,
1681                  (const uint8_t *)&rte_flow_item_mpls_mask,
1682                  sizeof(struct rte_flow_item_mpls), error);
1683         if (ret < 0)
1684                 return ret;
1685         return 0;
1686 #endif
1687         return rte_flow_error_set(error, ENOTSUP,
1688                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
1689                                   "MPLS is not supported by Verbs, please"
1690                                   " update.");
1691 }
1692
1693 static int
1694 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
1695                    const struct rte_flow_attr *attr __rte_unused,
1696                    const struct rte_flow_item items[] __rte_unused,
1697                    const struct rte_flow_action actions[] __rte_unused,
1698                    struct rte_flow_error *error __rte_unused)
1699 {
1700         rte_errno = ENOTSUP;
1701         return -rte_errno;
1702 }
1703
1704 static struct mlx5_flow *
1705 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
1706                   const struct rte_flow_item items[] __rte_unused,
1707                   const struct rte_flow_action actions[] __rte_unused,
1708                   struct rte_flow_error *error __rte_unused)
1709 {
1710         rte_errno = ENOTSUP;
1711         return NULL;
1712 }
1713
1714 static int
1715 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
1716                     struct mlx5_flow *dev_flow __rte_unused,
1717                     const struct rte_flow_attr *attr __rte_unused,
1718                     const struct rte_flow_item items[] __rte_unused,
1719                     const struct rte_flow_action actions[] __rte_unused,
1720                     struct rte_flow_error *error __rte_unused)
1721 {
1722         rte_errno = ENOTSUP;
1723         return -rte_errno;
1724 }
1725
1726 static int
1727 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
1728                 struct rte_flow *flow __rte_unused,
1729                 struct rte_flow_error *error __rte_unused)
1730 {
1731         rte_errno = ENOTSUP;
1732         return -rte_errno;
1733 }
1734
1735 static void
1736 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
1737                  struct rte_flow *flow __rte_unused)
1738 {
1739 }
1740
1741 static void
1742 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
1743                   struct rte_flow *flow __rte_unused)
1744 {
1745 }
1746
1747 static int
1748 flow_null_query(struct rte_eth_dev *dev __rte_unused,
1749                 struct rte_flow *flow __rte_unused,
1750                 const struct rte_flow_action *actions __rte_unused,
1751                 void *data __rte_unused,
1752                 struct rte_flow_error *error __rte_unused)
1753 {
1754         rte_errno = ENOTSUP;
1755         return -rte_errno;
1756 }
1757
1758 /* Void driver to protect from null pointer reference. */
1759 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
1760         .validate = flow_null_validate,
1761         .prepare = flow_null_prepare,
1762         .translate = flow_null_translate,
1763         .apply = flow_null_apply,
1764         .remove = flow_null_remove,
1765         .destroy = flow_null_destroy,
1766         .query = flow_null_query,
1767 };
1768
1769 /**
1770  * Select flow driver type according to flow attributes and device
1771  * configuration.
1772  *
1773  * @param[in] dev
1774  *   Pointer to the dev structure.
1775  * @param[in] attr
1776  *   Pointer to the flow attributes.
1777  *
1778  * @return
1779  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
1780  */
1781 static enum mlx5_flow_drv_type
1782 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
1783 {
1784         struct mlx5_priv *priv = dev->data->dev_private;
1785         enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
1786
1787         if (attr->transfer)
1788                 type = MLX5_FLOW_TYPE_TCF;
1789         else
1790                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
1791                                                  MLX5_FLOW_TYPE_VERBS;
1792         return type;
1793 }
1794
1795 #define flow_get_drv_ops(type) flow_drv_ops[type]
1796
1797 /**
1798  * Flow driver validation API. This abstracts calling driver specific functions.
1799  * The type of flow driver is determined according to flow attributes.
1800  *
1801  * @param[in] dev
1802  *   Pointer to the dev structure.
1803  * @param[in] attr
1804  *   Pointer to the flow attributes.
1805  * @param[in] items
1806  *   Pointer to the list of items.
1807  * @param[in] actions
1808  *   Pointer to the list of actions.
1809  * @param[out] error
1810  *   Pointer to the error structure.
1811  *
1812  * @return
1813  *   0 on success, a negative errno value otherwise and rte_errno is set.
1814  */
1815 static inline int
1816 flow_drv_validate(struct rte_eth_dev *dev,
1817                   const struct rte_flow_attr *attr,
1818                   const struct rte_flow_item items[],
1819                   const struct rte_flow_action actions[],
1820                   struct rte_flow_error *error)
1821 {
1822         const struct mlx5_flow_driver_ops *fops;
1823         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
1824
1825         fops = flow_get_drv_ops(type);
1826         return fops->validate(dev, attr, items, actions, error);
1827 }
1828
1829 /**
1830  * Flow driver preparation API. This abstracts calling driver specific
1831  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1832  * calculates the size of memory required for device flow, allocates the memory,
1833  * initializes the device flow and returns the pointer.
1834  *
1835  * @note
1836  *   This function initializes device flow structure such as dv, tcf or verbs in
1837  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
1838  *   rest. For example, adding returning device flow to flow->dev_flow list and
1839  *   setting backward reference to the flow should be done out of this function.
1840  *   layers field is not filled either.
1841  *
1842  * @param[in] attr
1843  *   Pointer to the flow attributes.
1844  * @param[in] items
1845  *   Pointer to the list of items.
1846  * @param[in] actions
1847  *   Pointer to the list of actions.
1848  * @param[out] error
1849  *   Pointer to the error structure.
1850  *
1851  * @return
1852  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
1853  */
1854 static inline struct mlx5_flow *
1855 flow_drv_prepare(const struct rte_flow *flow,
1856                  const struct rte_flow_attr *attr,
1857                  const struct rte_flow_item items[],
1858                  const struct rte_flow_action actions[],
1859                  struct rte_flow_error *error)
1860 {
1861         const struct mlx5_flow_driver_ops *fops;
1862         enum mlx5_flow_drv_type type = flow->drv_type;
1863
1864         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1865         fops = flow_get_drv_ops(type);
1866         return fops->prepare(attr, items, actions, error);
1867 }
1868
1869 /**
1870  * Flow driver translation API. This abstracts calling driver specific
1871  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1872  * translates a generic flow into a driver flow. flow_drv_prepare() must
1873  * precede.
1874  *
1875  * @note
1876  *   dev_flow->layers could be filled as a result of parsing during translation
1877  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
1878  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
1879  *   flow->actions could be overwritten even though all the expanded dev_flows
1880  *   have the same actions.
1881  *
1882  * @param[in] dev
1883  *   Pointer to the rte dev structure.
1884  * @param[in, out] dev_flow
1885  *   Pointer to the mlx5 flow.
1886  * @param[in] attr
1887  *   Pointer to the flow attributes.
1888  * @param[in] items
1889  *   Pointer to the list of items.
1890  * @param[in] actions
1891  *   Pointer to the list of actions.
1892  * @param[out] error
1893  *   Pointer to the error structure.
1894  *
1895  * @return
1896  *   0 on success, a negative errno value otherwise and rte_errno is set.
1897  */
1898 static inline int
1899 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
1900                    const struct rte_flow_attr *attr,
1901                    const struct rte_flow_item items[],
1902                    const struct rte_flow_action actions[],
1903                    struct rte_flow_error *error)
1904 {
1905         const struct mlx5_flow_driver_ops *fops;
1906         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
1907
1908         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1909         fops = flow_get_drv_ops(type);
1910         return fops->translate(dev, dev_flow, attr, items, actions, error);
1911 }
1912
1913 /**
1914  * Flow driver apply API. This abstracts calling driver specific functions.
1915  * Parent flow (rte_flow) should have driver type (drv_type). It applies
1916  * translated driver flows on to device. flow_drv_translate() must precede.
1917  *
1918  * @param[in] dev
1919  *   Pointer to Ethernet device structure.
1920  * @param[in, out] flow
1921  *   Pointer to flow structure.
1922  * @param[out] error
1923  *   Pointer to error structure.
1924  *
1925  * @return
1926  *   0 on success, a negative errno value otherwise and rte_errno is set.
1927  */
1928 static inline int
1929 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1930                struct rte_flow_error *error)
1931 {
1932         const struct mlx5_flow_driver_ops *fops;
1933         enum mlx5_flow_drv_type type = flow->drv_type;
1934
1935         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1936         fops = flow_get_drv_ops(type);
1937         return fops->apply(dev, flow, error);
1938 }
1939
1940 /**
1941  * Flow driver remove API. This abstracts calling driver specific functions.
1942  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1943  * on device. All the resources of the flow should be freed by calling
1944  * flow_drv_destroy().
1945  *
1946  * @param[in] dev
1947  *   Pointer to Ethernet device.
1948  * @param[in, out] flow
1949  *   Pointer to flow structure.
1950  */
1951 static inline void
1952 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1953 {
1954         const struct mlx5_flow_driver_ops *fops;
1955         enum mlx5_flow_drv_type type = flow->drv_type;
1956
1957         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1958         fops = flow_get_drv_ops(type);
1959         fops->remove(dev, flow);
1960 }
1961
1962 /**
1963  * Flow driver destroy API. This abstracts calling driver specific functions.
1964  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1965  * on device and releases resources of the flow.
1966  *
1967  * @param[in] dev
1968  *   Pointer to Ethernet device.
1969  * @param[in, out] flow
1970  *   Pointer to flow structure.
1971  */
1972 static inline void
1973 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1974 {
1975         const struct mlx5_flow_driver_ops *fops;
1976         enum mlx5_flow_drv_type type = flow->drv_type;
1977
1978         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1979         fops = flow_get_drv_ops(type);
1980         fops->destroy(dev, flow);
1981 }
1982
1983 /**
1984  * Validate a flow supported by the NIC.
1985  *
1986  * @see rte_flow_validate()
1987  * @see rte_flow_ops
1988  */
1989 int
1990 mlx5_flow_validate(struct rte_eth_dev *dev,
1991                    const struct rte_flow_attr *attr,
1992                    const struct rte_flow_item items[],
1993                    const struct rte_flow_action actions[],
1994                    struct rte_flow_error *error)
1995 {
1996         int ret;
1997
1998         ret = flow_drv_validate(dev, attr, items, actions, error);
1999         if (ret < 0)
2000                 return ret;
2001         return 0;
2002 }
2003
2004 /**
2005  * Get RSS action from the action list.
2006  *
2007  * @param[in] actions
2008  *   Pointer to the list of actions.
2009  *
2010  * @return
2011  *   Pointer to the RSS action if exist, else return NULL.
2012  */
2013 static const struct rte_flow_action_rss*
2014 flow_get_rss_action(const struct rte_flow_action actions[])
2015 {
2016         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2017                 switch (actions->type) {
2018                 case RTE_FLOW_ACTION_TYPE_RSS:
2019                         return (const struct rte_flow_action_rss *)
2020                                actions->conf;
2021                 default:
2022                         break;
2023                 }
2024         }
2025         return NULL;
2026 }
2027
2028 static unsigned int
2029 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
2030 {
2031         const struct rte_flow_item *item;
2032         unsigned int has_vlan = 0;
2033
2034         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2035                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
2036                         has_vlan = 1;
2037                         break;
2038                 }
2039         }
2040         if (has_vlan)
2041                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
2042                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
2043         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
2044                                MLX5_EXPANSION_ROOT_OUTER;
2045 }
2046
2047 /**
2048  * Create a flow and add it to @p list.
2049  *
2050  * @param dev
2051  *   Pointer to Ethernet device.
2052  * @param list
2053  *   Pointer to a TAILQ flow list.
2054  * @param[in] attr
2055  *   Flow rule attributes.
2056  * @param[in] items
2057  *   Pattern specification (list terminated by the END pattern item).
2058  * @param[in] actions
2059  *   Associated actions (list terminated by the END action).
2060  * @param[out] error
2061  *   Perform verbose error reporting if not NULL.
2062  *
2063  * @return
2064  *   A flow on success, NULL otherwise and rte_errno is set.
2065  */
2066 static struct rte_flow *
2067 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
2068                  const struct rte_flow_attr *attr,
2069                  const struct rte_flow_item items[],
2070                  const struct rte_flow_action actions[],
2071                  struct rte_flow_error *error)
2072 {
2073         struct rte_flow *flow = NULL;
2074         struct mlx5_flow *dev_flow;
2075         const struct rte_flow_action_rss *rss;
2076         union {
2077                 struct rte_flow_expand_rss buf;
2078                 uint8_t buffer[2048];
2079         } expand_buffer;
2080         struct rte_flow_expand_rss *buf = &expand_buffer.buf;
2081         int ret;
2082         uint32_t i;
2083         uint32_t flow_size;
2084
2085         ret = flow_drv_validate(dev, attr, items, actions, error);
2086         if (ret < 0)
2087                 return NULL;
2088         flow_size = sizeof(struct rte_flow);
2089         rss = flow_get_rss_action(actions);
2090         if (rss)
2091                 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
2092                                             sizeof(void *));
2093         else
2094                 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
2095         flow = rte_calloc(__func__, 1, flow_size, 0);
2096         flow->drv_type = flow_get_drv_type(dev, attr);
2097         flow->ingress = attr->ingress;
2098         assert(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
2099                flow->drv_type < MLX5_FLOW_TYPE_MAX);
2100         flow->queue = (void *)(flow + 1);
2101         LIST_INIT(&flow->dev_flows);
2102         if (rss && rss->types) {
2103                 unsigned int graph_root;
2104
2105                 graph_root = find_graph_root(items, rss->level);
2106                 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
2107                                           items, rss->types,
2108                                           mlx5_support_expansion,
2109                                           graph_root);
2110                 assert(ret > 0 &&
2111                        (unsigned int)ret < sizeof(expand_buffer.buffer));
2112         } else {
2113                 buf->entries = 1;
2114                 buf->entry[0].pattern = (void *)(uintptr_t)items;
2115         }
2116         for (i = 0; i < buf->entries; ++i) {
2117                 dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern,
2118                                             actions, error);
2119                 if (!dev_flow)
2120                         goto error;
2121                 dev_flow->flow = flow;
2122                 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
2123                 ret = flow_drv_translate(dev, dev_flow, attr,
2124                                          buf->entry[i].pattern,
2125                                          actions, error);
2126                 if (ret < 0)
2127                         goto error;
2128         }
2129         if (dev->data->dev_started) {
2130                 ret = flow_drv_apply(dev, flow, error);
2131                 if (ret < 0)
2132                         goto error;
2133         }
2134         TAILQ_INSERT_TAIL(list, flow, next);
2135         flow_rxq_flags_set(dev, flow);
2136         return flow;
2137 error:
2138         ret = rte_errno; /* Save rte_errno before cleanup. */
2139         assert(flow);
2140         flow_drv_destroy(dev, flow);
2141         rte_free(flow);
2142         rte_errno = ret; /* Restore rte_errno. */
2143         return NULL;
2144 }
2145
2146 /**
2147  * Create a flow.
2148  *
2149  * @see rte_flow_create()
2150  * @see rte_flow_ops
2151  */
2152 struct rte_flow *
2153 mlx5_flow_create(struct rte_eth_dev *dev,
2154                  const struct rte_flow_attr *attr,
2155                  const struct rte_flow_item items[],
2156                  const struct rte_flow_action actions[],
2157                  struct rte_flow_error *error)
2158 {
2159         struct mlx5_priv *priv = (struct mlx5_priv *)dev->data->dev_private;
2160
2161         return flow_list_create(dev, &priv->flows,
2162                                 attr, items, actions, error);
2163 }
2164
2165 /**
2166  * Destroy a flow in a list.
2167  *
2168  * @param dev
2169  *   Pointer to Ethernet device.
2170  * @param list
2171  *   Pointer to a TAILQ flow list.
2172  * @param[in] flow
2173  *   Flow to destroy.
2174  */
2175 static void
2176 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
2177                   struct rte_flow *flow)
2178 {
2179         /*
2180          * Update RX queue flags only if port is started, otherwise it is
2181          * already clean.
2182          */
2183         if (dev->data->dev_started)
2184                 flow_rxq_flags_trim(dev, flow);
2185         flow_drv_destroy(dev, flow);
2186         TAILQ_REMOVE(list, flow, next);
2187         rte_free(flow->fdir);
2188         rte_free(flow);
2189 }
2190
2191 /**
2192  * Destroy all flows.
2193  *
2194  * @param dev
2195  *   Pointer to Ethernet device.
2196  * @param list
2197  *   Pointer to a TAILQ flow list.
2198  */
2199 void
2200 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
2201 {
2202         while (!TAILQ_EMPTY(list)) {
2203                 struct rte_flow *flow;
2204
2205                 flow = TAILQ_FIRST(list);
2206                 flow_list_destroy(dev, list, flow);
2207         }
2208 }
2209
2210 /**
2211  * Remove all flows.
2212  *
2213  * @param dev
2214  *   Pointer to Ethernet device.
2215  * @param list
2216  *   Pointer to a TAILQ flow list.
2217  */
2218 void
2219 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
2220 {
2221         struct rte_flow *flow;
2222
2223         TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
2224                 flow_drv_remove(dev, flow);
2225         flow_rxq_flags_clear(dev);
2226 }
2227
2228 /**
2229  * Add all flows.
2230  *
2231  * @param dev
2232  *   Pointer to Ethernet device.
2233  * @param list
2234  *   Pointer to a TAILQ flow list.
2235  *
2236  * @return
2237  *   0 on success, a negative errno value otherwise and rte_errno is set.
2238  */
2239 int
2240 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
2241 {
2242         struct rte_flow *flow;
2243         struct rte_flow_error error;
2244         int ret = 0;
2245
2246         TAILQ_FOREACH(flow, list, next) {
2247                 ret = flow_drv_apply(dev, flow, &error);
2248                 if (ret < 0)
2249                         goto error;
2250                 flow_rxq_flags_set(dev, flow);
2251         }
2252         return 0;
2253 error:
2254         ret = rte_errno; /* Save rte_errno before cleanup. */
2255         mlx5_flow_stop(dev, list);
2256         rte_errno = ret; /* Restore rte_errno. */
2257         return -rte_errno;
2258 }
2259
2260 /**
2261  * Verify the flow list is empty
2262  *
2263  * @param dev
2264  *  Pointer to Ethernet device.
2265  *
2266  * @return the number of flows not released.
2267  */
2268 int
2269 mlx5_flow_verify(struct rte_eth_dev *dev)
2270 {
2271         struct mlx5_priv *priv = dev->data->dev_private;
2272         struct rte_flow *flow;
2273         int ret = 0;
2274
2275         TAILQ_FOREACH(flow, &priv->flows, next) {
2276                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
2277                         dev->data->port_id, (void *)flow);
2278                 ++ret;
2279         }
2280         return ret;
2281 }
2282
2283 /**
2284  * Enable a control flow configured from the control plane.
2285  *
2286  * @param dev
2287  *   Pointer to Ethernet device.
2288  * @param eth_spec
2289  *   An Ethernet flow spec to apply.
2290  * @param eth_mask
2291  *   An Ethernet flow mask to apply.
2292  * @param vlan_spec
2293  *   A VLAN flow spec to apply.
2294  * @param vlan_mask
2295  *   A VLAN flow mask to apply.
2296  *
2297  * @return
2298  *   0 on success, a negative errno value otherwise and rte_errno is set.
2299  */
2300 int
2301 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2302                     struct rte_flow_item_eth *eth_spec,
2303                     struct rte_flow_item_eth *eth_mask,
2304                     struct rte_flow_item_vlan *vlan_spec,
2305                     struct rte_flow_item_vlan *vlan_mask)
2306 {
2307         struct mlx5_priv *priv = dev->data->dev_private;
2308         const struct rte_flow_attr attr = {
2309                 .ingress = 1,
2310                 .priority = MLX5_FLOW_PRIO_RSVD,
2311         };
2312         struct rte_flow_item items[] = {
2313                 {
2314                         .type = RTE_FLOW_ITEM_TYPE_ETH,
2315                         .spec = eth_spec,
2316                         .last = NULL,
2317                         .mask = eth_mask,
2318                 },
2319                 {
2320                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2321                                               RTE_FLOW_ITEM_TYPE_END,
2322                         .spec = vlan_spec,
2323                         .last = NULL,
2324                         .mask = vlan_mask,
2325                 },
2326                 {
2327                         .type = RTE_FLOW_ITEM_TYPE_END,
2328                 },
2329         };
2330         uint16_t queue[priv->reta_idx_n];
2331         struct rte_flow_action_rss action_rss = {
2332                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
2333                 .level = 0,
2334                 .types = priv->rss_conf.rss_hf,
2335                 .key_len = priv->rss_conf.rss_key_len,
2336                 .queue_num = priv->reta_idx_n,
2337                 .key = priv->rss_conf.rss_key,
2338                 .queue = queue,
2339         };
2340         struct rte_flow_action actions[] = {
2341                 {
2342                         .type = RTE_FLOW_ACTION_TYPE_RSS,
2343                         .conf = &action_rss,
2344                 },
2345                 {
2346                         .type = RTE_FLOW_ACTION_TYPE_END,
2347                 },
2348         };
2349         struct rte_flow *flow;
2350         struct rte_flow_error error;
2351         unsigned int i;
2352
2353         if (!priv->reta_idx_n || !priv->rxqs_n) {
2354                 return 0;
2355         }
2356         for (i = 0; i != priv->reta_idx_n; ++i)
2357                 queue[i] = (*priv->reta_idx)[i];
2358         flow = flow_list_create(dev, &priv->ctrl_flows,
2359                                 &attr, items, actions, &error);
2360         if (!flow)
2361                 return -rte_errno;
2362         return 0;
2363 }
2364
2365 /**
2366  * Enable a flow control configured from the control plane.
2367  *
2368  * @param dev
2369  *   Pointer to Ethernet device.
2370  * @param eth_spec
2371  *   An Ethernet flow spec to apply.
2372  * @param eth_mask
2373  *   An Ethernet flow mask to apply.
2374  *
2375  * @return
2376  *   0 on success, a negative errno value otherwise and rte_errno is set.
2377  */
2378 int
2379 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2380                struct rte_flow_item_eth *eth_spec,
2381                struct rte_flow_item_eth *eth_mask)
2382 {
2383         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2384 }
2385
2386 /**
2387  * Destroy a flow.
2388  *
2389  * @see rte_flow_destroy()
2390  * @see rte_flow_ops
2391  */
2392 int
2393 mlx5_flow_destroy(struct rte_eth_dev *dev,
2394                   struct rte_flow *flow,
2395                   struct rte_flow_error *error __rte_unused)
2396 {
2397         struct mlx5_priv *priv = dev->data->dev_private;
2398
2399         flow_list_destroy(dev, &priv->flows, flow);
2400         return 0;
2401 }
2402
2403 /**
2404  * Destroy all flows.
2405  *
2406  * @see rte_flow_flush()
2407  * @see rte_flow_ops
2408  */
2409 int
2410 mlx5_flow_flush(struct rte_eth_dev *dev,
2411                 struct rte_flow_error *error __rte_unused)
2412 {
2413         struct mlx5_priv *priv = dev->data->dev_private;
2414
2415         mlx5_flow_list_flush(dev, &priv->flows);
2416         return 0;
2417 }
2418
2419 /**
2420  * Isolated mode.
2421  *
2422  * @see rte_flow_isolate()
2423  * @see rte_flow_ops
2424  */
2425 int
2426 mlx5_flow_isolate(struct rte_eth_dev *dev,
2427                   int enable,
2428                   struct rte_flow_error *error)
2429 {
2430         struct mlx5_priv *priv = dev->data->dev_private;
2431
2432         if (dev->data->dev_started) {
2433                 rte_flow_error_set(error, EBUSY,
2434                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2435                                    NULL,
2436                                    "port must be stopped first");
2437                 return -rte_errno;
2438         }
2439         priv->isolated = !!enable;
2440         if (enable)
2441                 dev->dev_ops = &mlx5_dev_ops_isolate;
2442         else
2443                 dev->dev_ops = &mlx5_dev_ops;
2444         return 0;
2445 }
2446
2447 /**
2448  * Query a flow.
2449  *
2450  * @see rte_flow_query()
2451  * @see rte_flow_ops
2452  */
2453 static int
2454 flow_drv_query(struct rte_eth_dev *dev,
2455                struct rte_flow *flow,
2456                const struct rte_flow_action *actions,
2457                void *data,
2458                struct rte_flow_error *error)
2459 {
2460         const struct mlx5_flow_driver_ops *fops;
2461         enum mlx5_flow_drv_type ftype = flow->drv_type;
2462
2463         assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
2464         fops = flow_get_drv_ops(ftype);
2465
2466         return fops->query(dev, flow, actions, data, error);
2467 }
2468
2469 /**
2470  * Query a flow.
2471  *
2472  * @see rte_flow_query()
2473  * @see rte_flow_ops
2474  */
2475 int
2476 mlx5_flow_query(struct rte_eth_dev *dev,
2477                 struct rte_flow *flow,
2478                 const struct rte_flow_action *actions,
2479                 void *data,
2480                 struct rte_flow_error *error)
2481 {
2482         int ret;
2483
2484         ret = flow_drv_query(dev, flow, actions, data, error);
2485         if (ret < 0)
2486                 return ret;
2487         return 0;
2488 }
2489
2490 /**
2491  * Convert a flow director filter to a generic flow.
2492  *
2493  * @param dev
2494  *   Pointer to Ethernet device.
2495  * @param fdir_filter
2496  *   Flow director filter to add.
2497  * @param attributes
2498  *   Generic flow parameters structure.
2499  *
2500  * @return
2501  *   0 on success, a negative errno value otherwise and rte_errno is set.
2502  */
2503 static int
2504 flow_fdir_filter_convert(struct rte_eth_dev *dev,
2505                          const struct rte_eth_fdir_filter *fdir_filter,
2506                          struct mlx5_fdir *attributes)
2507 {
2508         struct mlx5_priv *priv = dev->data->dev_private;
2509         const struct rte_eth_fdir_input *input = &fdir_filter->input;
2510         const struct rte_eth_fdir_masks *mask =
2511                 &dev->data->dev_conf.fdir_conf.mask;
2512
2513         /* Validate queue number. */
2514         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2515                 DRV_LOG(ERR, "port %u invalid queue number %d",
2516                         dev->data->port_id, fdir_filter->action.rx_queue);
2517                 rte_errno = EINVAL;
2518                 return -rte_errno;
2519         }
2520         attributes->attr.ingress = 1;
2521         attributes->items[0] = (struct rte_flow_item) {
2522                 .type = RTE_FLOW_ITEM_TYPE_ETH,
2523                 .spec = &attributes->l2,
2524                 .mask = &attributes->l2_mask,
2525         };
2526         switch (fdir_filter->action.behavior) {
2527         case RTE_ETH_FDIR_ACCEPT:
2528                 attributes->actions[0] = (struct rte_flow_action){
2529                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2530                         .conf = &attributes->queue,
2531                 };
2532                 break;
2533         case RTE_ETH_FDIR_REJECT:
2534                 attributes->actions[0] = (struct rte_flow_action){
2535                         .type = RTE_FLOW_ACTION_TYPE_DROP,
2536                 };
2537                 break;
2538         default:
2539                 DRV_LOG(ERR, "port %u invalid behavior %d",
2540                         dev->data->port_id,
2541                         fdir_filter->action.behavior);
2542                 rte_errno = ENOTSUP;
2543                 return -rte_errno;
2544         }
2545         attributes->queue.index = fdir_filter->action.rx_queue;
2546         /* Handle L3. */
2547         switch (fdir_filter->input.flow_type) {
2548         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2549         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2550         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2551                 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2552                         .src_addr = input->flow.ip4_flow.src_ip,
2553                         .dst_addr = input->flow.ip4_flow.dst_ip,
2554                         .time_to_live = input->flow.ip4_flow.ttl,
2555                         .type_of_service = input->flow.ip4_flow.tos,
2556                 };
2557                 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
2558                         .src_addr = mask->ipv4_mask.src_ip,
2559                         .dst_addr = mask->ipv4_mask.dst_ip,
2560                         .time_to_live = mask->ipv4_mask.ttl,
2561                         .type_of_service = mask->ipv4_mask.tos,
2562                         .next_proto_id = mask->ipv4_mask.proto,
2563                 };
2564                 attributes->items[1] = (struct rte_flow_item){
2565                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
2566                         .spec = &attributes->l3,
2567                         .mask = &attributes->l3_mask,
2568                 };
2569                 break;
2570         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2571         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2572         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2573                 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2574                         .hop_limits = input->flow.ipv6_flow.hop_limits,
2575                         .proto = input->flow.ipv6_flow.proto,
2576                 };
2577
2578                 memcpy(attributes->l3.ipv6.hdr.src_addr,
2579                        input->flow.ipv6_flow.src_ip,
2580                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2581                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2582                        input->flow.ipv6_flow.dst_ip,
2583                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2584                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
2585                        mask->ipv6_mask.src_ip,
2586                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2587                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
2588                        mask->ipv6_mask.dst_ip,
2589                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2590                 attributes->items[1] = (struct rte_flow_item){
2591                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
2592                         .spec = &attributes->l3,
2593                         .mask = &attributes->l3_mask,
2594                 };
2595                 break;
2596         default:
2597                 DRV_LOG(ERR, "port %u invalid flow type%d",
2598                         dev->data->port_id, fdir_filter->input.flow_type);
2599                 rte_errno = ENOTSUP;
2600                 return -rte_errno;
2601         }
2602         /* Handle L4. */
2603         switch (fdir_filter->input.flow_type) {
2604         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2605                 attributes->l4.udp.hdr = (struct udp_hdr){
2606                         .src_port = input->flow.udp4_flow.src_port,
2607                         .dst_port = input->flow.udp4_flow.dst_port,
2608                 };
2609                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2610                         .src_port = mask->src_port_mask,
2611                         .dst_port = mask->dst_port_mask,
2612                 };
2613                 attributes->items[2] = (struct rte_flow_item){
2614                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2615                         .spec = &attributes->l4,
2616                         .mask = &attributes->l4_mask,
2617                 };
2618                 break;
2619         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2620                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2621                         .src_port = input->flow.tcp4_flow.src_port,
2622                         .dst_port = input->flow.tcp4_flow.dst_port,
2623                 };
2624                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2625                         .src_port = mask->src_port_mask,
2626                         .dst_port = mask->dst_port_mask,
2627                 };
2628                 attributes->items[2] = (struct rte_flow_item){
2629                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2630                         .spec = &attributes->l4,
2631                         .mask = &attributes->l4_mask,
2632                 };
2633                 break;
2634         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2635                 attributes->l4.udp.hdr = (struct udp_hdr){
2636                         .src_port = input->flow.udp6_flow.src_port,
2637                         .dst_port = input->flow.udp6_flow.dst_port,
2638                 };
2639                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2640                         .src_port = mask->src_port_mask,
2641                         .dst_port = mask->dst_port_mask,
2642                 };
2643                 attributes->items[2] = (struct rte_flow_item){
2644                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2645                         .spec = &attributes->l4,
2646                         .mask = &attributes->l4_mask,
2647                 };
2648                 break;
2649         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2650                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2651                         .src_port = input->flow.tcp6_flow.src_port,
2652                         .dst_port = input->flow.tcp6_flow.dst_port,
2653                 };
2654                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2655                         .src_port = mask->src_port_mask,
2656                         .dst_port = mask->dst_port_mask,
2657                 };
2658                 attributes->items[2] = (struct rte_flow_item){
2659                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2660                         .spec = &attributes->l4,
2661                         .mask = &attributes->l4_mask,
2662                 };
2663                 break;
2664         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2665         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2666                 break;
2667         default:
2668                 DRV_LOG(ERR, "port %u invalid flow type%d",
2669                         dev->data->port_id, fdir_filter->input.flow_type);
2670                 rte_errno = ENOTSUP;
2671                 return -rte_errno;
2672         }
2673         return 0;
2674 }
2675
2676 #define FLOW_FDIR_CMP(f1, f2, fld) \
2677         memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
2678
2679 /**
2680  * Compare two FDIR flows. If items and actions are identical, the two flows are
2681  * regarded as same.
2682  *
2683  * @param dev
2684  *   Pointer to Ethernet device.
2685  * @param f1
2686  *   FDIR flow to compare.
2687  * @param f2
2688  *   FDIR flow to compare.
2689  *
2690  * @return
2691  *   Zero on match, 1 otherwise.
2692  */
2693 static int
2694 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
2695 {
2696         if (FLOW_FDIR_CMP(f1, f2, attr) ||
2697             FLOW_FDIR_CMP(f1, f2, l2) ||
2698             FLOW_FDIR_CMP(f1, f2, l2_mask) ||
2699             FLOW_FDIR_CMP(f1, f2, l3) ||
2700             FLOW_FDIR_CMP(f1, f2, l3_mask) ||
2701             FLOW_FDIR_CMP(f1, f2, l4) ||
2702             FLOW_FDIR_CMP(f1, f2, l4_mask) ||
2703             FLOW_FDIR_CMP(f1, f2, actions[0].type))
2704                 return 1;
2705         if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
2706             FLOW_FDIR_CMP(f1, f2, queue))
2707                 return 1;
2708         return 0;
2709 }
2710
2711 /**
2712  * Search device flow list to find out a matched FDIR flow.
2713  *
2714  * @param dev
2715  *   Pointer to Ethernet device.
2716  * @param fdir_flow
2717  *   FDIR flow to lookup.
2718  *
2719  * @return
2720  *   Pointer of flow if found, NULL otherwise.
2721  */
2722 static struct rte_flow *
2723 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
2724 {
2725         struct mlx5_priv *priv = dev->data->dev_private;
2726         struct rte_flow *flow = NULL;
2727
2728         assert(fdir_flow);
2729         TAILQ_FOREACH(flow, &priv->flows, next) {
2730                 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) {
2731                         DRV_LOG(DEBUG, "port %u found FDIR flow %p",
2732                                 dev->data->port_id, (void *)flow);
2733                         break;
2734                 }
2735         }
2736         return flow;
2737 }
2738
2739 /**
2740  * Add new flow director filter and store it in list.
2741  *
2742  * @param dev
2743  *   Pointer to Ethernet device.
2744  * @param fdir_filter
2745  *   Flow director filter to add.
2746  *
2747  * @return
2748  *   0 on success, a negative errno value otherwise and rte_errno is set.
2749  */
2750 static int
2751 flow_fdir_filter_add(struct rte_eth_dev *dev,
2752                      const struct rte_eth_fdir_filter *fdir_filter)
2753 {
2754         struct mlx5_priv *priv = dev->data->dev_private;
2755         struct mlx5_fdir *fdir_flow;
2756         struct rte_flow *flow;
2757         int ret;
2758
2759         fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0);
2760         if (!fdir_flow) {
2761                 rte_errno = ENOMEM;
2762                 return -rte_errno;
2763         }
2764         ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
2765         if (ret)
2766                 goto error;
2767         flow = flow_fdir_filter_lookup(dev, fdir_flow);
2768         if (flow) {
2769                 rte_errno = EEXIST;
2770                 goto error;
2771         }
2772         flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
2773                                 fdir_flow->items, fdir_flow->actions, NULL);
2774         if (!flow)
2775                 goto error;
2776         assert(!flow->fdir);
2777         flow->fdir = fdir_flow;
2778         DRV_LOG(DEBUG, "port %u created FDIR flow %p",
2779                 dev->data->port_id, (void *)flow);
2780         return 0;
2781 error:
2782         rte_free(fdir_flow);
2783         return -rte_errno;
2784 }
2785
2786 /**
2787  * Delete specific filter.
2788  *
2789  * @param dev
2790  *   Pointer to Ethernet device.
2791  * @param fdir_filter
2792  *   Filter to be deleted.
2793  *
2794  * @return
2795  *   0 on success, a negative errno value otherwise and rte_errno is set.
2796  */
2797 static int
2798 flow_fdir_filter_delete(struct rte_eth_dev *dev,
2799                         const struct rte_eth_fdir_filter *fdir_filter)
2800 {
2801         struct mlx5_priv *priv = dev->data->dev_private;
2802         struct rte_flow *flow;
2803         struct mlx5_fdir fdir_flow = {
2804                 .attr.group = 0,
2805         };
2806         int ret;
2807
2808         ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
2809         if (ret)
2810                 return -rte_errno;
2811         flow = flow_fdir_filter_lookup(dev, &fdir_flow);
2812         if (!flow) {
2813                 rte_errno = ENOENT;
2814                 return -rte_errno;
2815         }
2816         flow_list_destroy(dev, &priv->flows, flow);
2817         DRV_LOG(DEBUG, "port %u deleted FDIR flow %p",
2818                 dev->data->port_id, (void *)flow);
2819         return 0;
2820 }
2821
2822 /**
2823  * Update queue for specific filter.
2824  *
2825  * @param dev
2826  *   Pointer to Ethernet device.
2827  * @param fdir_filter
2828  *   Filter to be updated.
2829  *
2830  * @return
2831  *   0 on success, a negative errno value otherwise and rte_errno is set.
2832  */
2833 static int
2834 flow_fdir_filter_update(struct rte_eth_dev *dev,
2835                         const struct rte_eth_fdir_filter *fdir_filter)
2836 {
2837         int ret;
2838
2839         ret = flow_fdir_filter_delete(dev, fdir_filter);
2840         if (ret)
2841                 return ret;
2842         return flow_fdir_filter_add(dev, fdir_filter);
2843 }
2844
2845 /**
2846  * Flush all filters.
2847  *
2848  * @param dev
2849  *   Pointer to Ethernet device.
2850  */
2851 static void
2852 flow_fdir_filter_flush(struct rte_eth_dev *dev)
2853 {
2854         struct mlx5_priv *priv = dev->data->dev_private;
2855
2856         mlx5_flow_list_flush(dev, &priv->flows);
2857 }
2858
2859 /**
2860  * Get flow director information.
2861  *
2862  * @param dev
2863  *   Pointer to Ethernet device.
2864  * @param[out] fdir_info
2865  *   Resulting flow director information.
2866  */
2867 static void
2868 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
2869 {
2870         struct rte_eth_fdir_masks *mask =
2871                 &dev->data->dev_conf.fdir_conf.mask;
2872
2873         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
2874         fdir_info->guarant_spc = 0;
2875         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2876         fdir_info->max_flexpayload = 0;
2877         fdir_info->flow_types_mask[0] = 0;
2878         fdir_info->flex_payload_unit = 0;
2879         fdir_info->max_flex_payload_segment_num = 0;
2880         fdir_info->flex_payload_limit = 0;
2881         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2882 }
2883
2884 /**
2885  * Deal with flow director operations.
2886  *
2887  * @param dev
2888  *   Pointer to Ethernet device.
2889  * @param filter_op
2890  *   Operation to perform.
2891  * @param arg
2892  *   Pointer to operation-specific structure.
2893  *
2894  * @return
2895  *   0 on success, a negative errno value otherwise and rte_errno is set.
2896  */
2897 static int
2898 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
2899                     void *arg)
2900 {
2901         enum rte_fdir_mode fdir_mode =
2902                 dev->data->dev_conf.fdir_conf.mode;
2903
2904         if (filter_op == RTE_ETH_FILTER_NOP)
2905                 return 0;
2906         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2907             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2908                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
2909                         dev->data->port_id, fdir_mode);
2910                 rte_errno = EINVAL;
2911                 return -rte_errno;
2912         }
2913         switch (filter_op) {
2914         case RTE_ETH_FILTER_ADD:
2915                 return flow_fdir_filter_add(dev, arg);
2916         case RTE_ETH_FILTER_UPDATE:
2917                 return flow_fdir_filter_update(dev, arg);
2918         case RTE_ETH_FILTER_DELETE:
2919                 return flow_fdir_filter_delete(dev, arg);
2920         case RTE_ETH_FILTER_FLUSH:
2921                 flow_fdir_filter_flush(dev);
2922                 break;
2923         case RTE_ETH_FILTER_INFO:
2924                 flow_fdir_info_get(dev, arg);
2925                 break;
2926         default:
2927                 DRV_LOG(DEBUG, "port %u unknown operation %u",
2928                         dev->data->port_id, filter_op);
2929                 rte_errno = EINVAL;
2930                 return -rte_errno;
2931         }
2932         return 0;
2933 }
2934
2935 /**
2936  * Manage filter operations.
2937  *
2938  * @param dev
2939  *   Pointer to Ethernet device structure.
2940  * @param filter_type
2941  *   Filter type.
2942  * @param filter_op
2943  *   Operation to perform.
2944  * @param arg
2945  *   Pointer to operation-specific structure.
2946  *
2947  * @return
2948  *   0 on success, a negative errno value otherwise and rte_errno is set.
2949  */
2950 int
2951 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2952                      enum rte_filter_type filter_type,
2953                      enum rte_filter_op filter_op,
2954                      void *arg)
2955 {
2956         switch (filter_type) {
2957         case RTE_ETH_FILTER_GENERIC:
2958                 if (filter_op != RTE_ETH_FILTER_GET) {
2959                         rte_errno = EINVAL;
2960                         return -rte_errno;
2961                 }
2962                 *(const void **)arg = &mlx5_flow_ops;
2963                 return 0;
2964         case RTE_ETH_FILTER_FDIR:
2965                 return flow_fdir_ctrl_func(dev, filter_op, arg);
2966         default:
2967                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
2968                         dev->data->port_id, filter_type);
2969                 rte_errno = ENOTSUP;
2970                 return -rte_errno;
2971         }
2972         return 0;
2973 }