drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <netinet/in.h>
   7 #include <sys/queue.h>
   8 #include <stdalign.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11
  12 /* Verbs header. */
  13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  14 #ifdef PEDANTIC
  15 #pragma GCC diagnostic ignored "-Wpedantic"
  16 #endif
  17 #include <infiniband/verbs.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_common.h>
  23 #include <rte_ether.h>
  24 #include <rte_eth_ctrl.h>
  25 #include <rte_ethdev_driver.h>
  26 #include <rte_flow.h>
  27 #include <rte_flow_driver.h>
  28 #include <rte_malloc.h>
  29 #include <rte_ip.h>
  30
  31 #include "mlx5.h"
  32 #include "mlx5_defs.h"
  33 #include "mlx5_prm.h"
  34 #include "mlx5_glue.h"
  35 #include "mlx5_flow.h"
  36
  37 /* Dev ops structure defined in mlx5.c */
  38 extern const struct eth_dev_ops mlx5_dev_ops;
  39 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
  40
  41 /** Device flow drivers. */
  42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
  44 #endif
  45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  46
  47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  48
  49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  50         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  52         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  53 #endif
  54         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  55         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  56 };
  57
  58 enum mlx5_expansion {
  59         MLX5_EXPANSION_ROOT,
  60         MLX5_EXPANSION_ROOT_OUTER,
  61         MLX5_EXPANSION_ROOT_ETH_VLAN,
  62         MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
  63         MLX5_EXPANSION_OUTER_ETH,
  64         MLX5_EXPANSION_OUTER_ETH_VLAN,
  65         MLX5_EXPANSION_OUTER_VLAN,
  66         MLX5_EXPANSION_OUTER_IPV4,
  67         MLX5_EXPANSION_OUTER_IPV4_UDP,
  68         MLX5_EXPANSION_OUTER_IPV4_TCP,
  69         MLX5_EXPANSION_OUTER_IPV6,
  70         MLX5_EXPANSION_OUTER_IPV6_UDP,
  71         MLX5_EXPANSION_OUTER_IPV6_TCP,
  72         MLX5_EXPANSION_VXLAN,
  73         MLX5_EXPANSION_VXLAN_GPE,
  74         MLX5_EXPANSION_GRE,
  75         MLX5_EXPANSION_MPLS,
  76         MLX5_EXPANSION_ETH,
  77         MLX5_EXPANSION_ETH_VLAN,
  78         MLX5_EXPANSION_VLAN,
  79         MLX5_EXPANSION_IPV4,
  80         MLX5_EXPANSION_IPV4_UDP,
  81         MLX5_EXPANSION_IPV4_TCP,
  82         MLX5_EXPANSION_IPV6,
  83         MLX5_EXPANSION_IPV6_UDP,
  84         MLX5_EXPANSION_IPV6_TCP,
  85 };
  86
  87 /** Supported expansion of items. */
  88 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
  89         [MLX5_EXPANSION_ROOT] = {
  90                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
  91                                                  MLX5_EXPANSION_IPV4,
  92                                                  MLX5_EXPANSION_IPV6),
  93                 .type = RTE_FLOW_ITEM_TYPE_END,
  94         },
  95         [MLX5_EXPANSION_ROOT_OUTER] = {
  96                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
  97                                                  MLX5_EXPANSION_OUTER_IPV4,
  98                                                  MLX5_EXPANSION_OUTER_IPV6),
  99                 .type = RTE_FLOW_ITEM_TYPE_END,
 100         },
 101         [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
 102                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
 103                 .type = RTE_FLOW_ITEM_TYPE_END,
 104         },
 105         [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
 106                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
 107                 .type = RTE_FLOW_ITEM_TYPE_END,
 108         },
 109         [MLX5_EXPANSION_OUTER_ETH] = {
 110                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 111                                                  MLX5_EXPANSION_OUTER_IPV6,
 112                                                  MLX5_EXPANSION_MPLS),
 113                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 114                 .rss_types = 0,
 115         },
 116         [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 117                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 118                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 119                 .rss_types = 0,
 120         },
 121         [MLX5_EXPANSION_OUTER_VLAN] = {
 122                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 123                                                  MLX5_EXPANSION_OUTER_IPV6),
 124                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 125         },
 126         [MLX5_EXPANSION_OUTER_IPV4] = {
 127                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 128                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 129                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 130                          MLX5_EXPANSION_GRE),
 131                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 132                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 133                         ETH_RSS_NONFRAG_IPV4_OTHER,
 134         },
 135         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 136                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 137                                                  MLX5_EXPANSION_VXLAN_GPE),
 138                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 139                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 140         },
 141         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 142                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 143                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 144         },
 145         [MLX5_EXPANSION_OUTER_IPV6] = {
 146                 .next = RTE_FLOW_EXPAND_RSS_NEXT
 147                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 148                          MLX5_EXPANSION_OUTER_IPV6_TCP),
 149                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 150                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 151                         ETH_RSS_NONFRAG_IPV6_OTHER,
 152         },
 153         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 154                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 155                                                  MLX5_EXPANSION_VXLAN_GPE),
 156                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 157                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 158         },
 159         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 160                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 161                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 162         },
 163         [MLX5_EXPANSION_VXLAN] = {
 164                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 165                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 166         },
 167         [MLX5_EXPANSION_VXLAN_GPE] = {
 168                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 169                                                  MLX5_EXPANSION_IPV4,
 170                                                  MLX5_EXPANSION_IPV6),
 171                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 172         },
 173         [MLX5_EXPANSION_GRE] = {
 174                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 175                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 176         },
 177         [MLX5_EXPANSION_MPLS] = {
 178                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 179                                                  MLX5_EXPANSION_IPV6),
 180                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 181         },
 182         [MLX5_EXPANSION_ETH] = {
 183                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 184                                                  MLX5_EXPANSION_IPV6),
 185                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 186         },
 187         [MLX5_EXPANSION_ETH_VLAN] = {
 188                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 189                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 190         },
 191         [MLX5_EXPANSION_VLAN] = {
 192                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 193                                                  MLX5_EXPANSION_IPV6),
 194                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 195         },
 196         [MLX5_EXPANSION_IPV4] = {
 197                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 198                                                  MLX5_EXPANSION_IPV4_TCP),
 199                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 200                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 201                         ETH_RSS_NONFRAG_IPV4_OTHER,
 202         },
 203         [MLX5_EXPANSION_IPV4_UDP] = {
 204                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 205                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 206         },
 207         [MLX5_EXPANSION_IPV4_TCP] = {
 208                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 209                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 210         },
 211         [MLX5_EXPANSION_IPV6] = {
 212                 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 213                                                  MLX5_EXPANSION_IPV6_TCP),
 214                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 215                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 216                         ETH_RSS_NONFRAG_IPV6_OTHER,
 217         },
 218         [MLX5_EXPANSION_IPV6_UDP] = {
 219                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 220                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 221         },
 222         [MLX5_EXPANSION_IPV6_TCP] = {
 223                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 224                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 225         },
 226 };
 227
 228 static const struct rte_flow_ops mlx5_flow_ops = {
 229         .validate = mlx5_flow_validate,
 230         .create = mlx5_flow_create,
 231         .destroy = mlx5_flow_destroy,
 232         .flush = mlx5_flow_flush,
 233         .isolate = mlx5_flow_isolate,
 234         .query = mlx5_flow_query,
 235 };
 236
 237 /* Convert FDIR request to Generic flow. */
 238 struct mlx5_fdir {
 239         struct rte_flow_attr attr;
 240         struct rte_flow_action actions[2];
 241         struct rte_flow_item items[4];
 242         struct rte_flow_item_eth l2;
 243         struct rte_flow_item_eth l2_mask;
 244         union {
 245                 struct rte_flow_item_ipv4 ipv4;
 246                 struct rte_flow_item_ipv6 ipv6;
 247         } l3;
 248         union {
 249                 struct rte_flow_item_ipv4 ipv4;
 250                 struct rte_flow_item_ipv6 ipv6;
 251         } l3_mask;
 252         union {
 253                 struct rte_flow_item_udp udp;
 254                 struct rte_flow_item_tcp tcp;
 255         } l4;
 256         union {
 257                 struct rte_flow_item_udp udp;
 258                 struct rte_flow_item_tcp tcp;
 259         } l4_mask;
 260         struct rte_flow_action_queue queue;
 261 };
 262
 263 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
 264 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
 265         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
 266 };
 267
 268 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
 269 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 270         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
 271         { 9, 10, 11 }, { 12, 13, 14 },
 272 };
 273
 274 /* Tunnel information. */
 275 struct mlx5_flow_tunnel_info {
 276         uint32_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 277         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 278 };
 279
 280 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 281         {
 282                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 283                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 284         },
 285         {
 286                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 287                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 288         },
 289         {
 290                 .tunnel = MLX5_FLOW_LAYER_GRE,
 291                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 292         },
 293         {
 294                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 295                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
 296         },
 297         {
 298                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 299                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 300         },
 301 };
 302
 303 /**
 304  * Discover the maximum number of priority available.
 305  *
 306  * @param[in] dev
 307  *   Pointer to the Ethernet device structure.
 308  *
 309  * @return
 310  *   number of supported flow priority on success, a negative errno
 311  *   value otherwise and rte_errno is set.
 312  */
 313 int
 314 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 315 {
 316         struct {
 317                 struct ibv_flow_attr attr;
 318                 struct ibv_flow_spec_eth eth;
 319                 struct ibv_flow_spec_action_drop drop;
 320         } flow_attr = {
 321                 .attr = {
 322                         .num_of_specs = 2,
 323                 },
 324                 .eth = {
 325                         .type = IBV_FLOW_SPEC_ETH,
 326                         .size = sizeof(struct ibv_flow_spec_eth),
 327                 },
 328                 .drop = {
 329                         .size = sizeof(struct ibv_flow_spec_action_drop),
 330                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 331                 },
 332         };
 333         struct ibv_flow *flow;
 334         struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 335         uint16_t vprio[] = { 8, 16 };
 336         int i;
 337         int priority = 0;
 338
 339         if (!drop) {
 340                 rte_errno = ENOTSUP;
 341                 return -rte_errno;
 342         }
 343         for (i = 0; i != RTE_DIM(vprio); i++) {
 344                 flow_attr.attr.priority = vprio[i] - 1;
 345                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 346                 if (!flow)
 347                         break;
 348                 claim_zero(mlx5_glue->destroy_flow(flow));
 349                 priority = vprio[i];
 350         }
 351         switch (priority) {
 352         case 8:
 353                 priority = RTE_DIM(priority_map_3);
 354                 break;
 355         case 16:
 356                 priority = RTE_DIM(priority_map_5);
 357                 break;
 358         default:
 359                 rte_errno = ENOTSUP;
 360                 DRV_LOG(ERR,
 361                         "port %u verbs maximum priority: %d expected 8/16",
 362                         dev->data->port_id, vprio[i]);
 363                 return -rte_errno;
 364         }
 365         mlx5_hrxq_drop_release(dev);
 366         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 367                 dev->data->port_id, priority);
 368         return priority;
 369 }
 370
 371 /**
 372  * Adjust flow priority based on the highest layer and the request priority.
 373  *
 374  * @param[in] dev
 375  *   Pointer to the Ethernet device structure.
 376  * @param[in] priority
 377  *   The rule base priority.
 378  * @param[in] subpriority
 379  *   The priority based on the items.
 380  *
 381  * @return
 382  *   The new priority.
 383  */
 384 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 385                                    uint32_t subpriority)
 386 {
 387         uint32_t res = 0;
 388         struct priv *priv = dev->data->dev_private;
 389
 390         switch (priv->config.flow_prio) {
 391         case RTE_DIM(priority_map_3):
 392                 res = priority_map_3[priority][subpriority];
 393                 break;
 394         case RTE_DIM(priority_map_5):
 395                 res = priority_map_5[priority][subpriority];
 396                 break;
 397         }
 398         return  res;
 399 }
 400
 401 /**
 402  * Verify the @p item specifications (spec, last, mask) are compatible with the
 403  * NIC capabilities.
 404  *
 405  * @param[in] item
 406  *   Item specification.
 407  * @param[in] mask
 408  *   @p item->mask or flow default bit-masks.
 409  * @param[in] nic_mask
 410  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 411  * @param[in] size
 412  *   Bit-masks size in bytes.
 413  * @param[out] error
 414  *   Pointer to error structure.
 415  *
 416  * @return
 417  *   0 on success, a negative errno value otherwise and rte_errno is set.
 418  */
 419 static int
 420 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 421                           const uint8_t *mask,
 422                           const uint8_t *nic_mask,
 423                           unsigned int size,
 424                           struct rte_flow_error *error)
 425 {
 426         unsigned int i;
 427
 428         assert(nic_mask);
 429         for (i = 0; i < size; ++i)
 430                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
 431                         return rte_flow_error_set(error, ENOTSUP,
 432                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 433                                                   item,
 434                                                   "mask enables non supported"
 435                                                   " bits");
 436         if (!item->spec && (item->mask || item->last))
 437                 return rte_flow_error_set(error, EINVAL,
 438                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 439                                           "mask/last without a spec is not"
 440                                           " supported");
 441         if (item->spec && item->last) {
 442                 uint8_t spec[size];
 443                 uint8_t last[size];
 444                 unsigned int i;
 445                 int ret;
 446
 447                 for (i = 0; i < size; ++i) {
 448                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
 449                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 450                 }
 451                 ret = memcmp(spec, last, size);
 452                 if (ret != 0)
 453                         return rte_flow_error_set(error, ENOTSUP,
 454                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 455                                                   item,
 456                                                   "range is not supported");
 457         }
 458         return 0;
 459 }
 460
 461 /**
 462  * Adjust the hash fields according to the @p flow information.
 463  *
 464  * @param[in] dev_flow.
 465  *   Pointer to the mlx5_flow.
 466  * @param[in] tunnel
 467  *   1 when the hash field is for a tunnel item.
 468  * @param[in] layer_types
 469  *   ETH_RSS_* types.
 470  * @param[in] hash_fields
 471  *   Item hash fields.
 472  *
 473  * @return
 474  *   The hash fileds that should be used.
 475  */
 476 uint64_t
 477 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
 478                             int tunnel __rte_unused, uint32_t layer_types,
 479                             uint64_t hash_fields)
 480 {
 481         struct rte_flow *flow = dev_flow->flow;
 482 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 483         int rss_request_inner = flow->rss.level >= 2;
 484
 485         /* Check RSS hash level for tunnel. */
 486         if (tunnel && rss_request_inner)
 487                 hash_fields |= IBV_RX_HASH_INNER;
 488         else if (tunnel || rss_request_inner)
 489                 return 0;
 490 #endif
 491         /* Check if requested layer matches RSS hash fields. */
 492         if (!(flow->rss.types & layer_types))
 493                 return 0;
 494         return hash_fields;
 495 }
 496
 497 /**
 498  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
 499  * if several tunnel rules are used on this queue, the tunnel ptype will be
 500  * cleared.
 501  *
 502  * @param rxq_ctrl
 503  *   Rx queue to update.
 504  */
 505 static void
 506 mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 507 {
 508         unsigned int i;
 509         uint32_t tunnel_ptype = 0;
 510
 511         /* Look up for the ptype to use. */
 512         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
 513                 if (!rxq_ctrl->flow_tunnels_n[i])
 514                         continue;
 515                 if (!tunnel_ptype) {
 516                         tunnel_ptype = tunnels_info[i].ptype;
 517                 } else {
 518                         tunnel_ptype = 0;
 519                         break;
 520                 }
 521         }
 522         rxq_ctrl->rxq.tunnel = tunnel_ptype;
 523 }
 524
 525 /**
 526  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
 527  *
 528  * @param[in] dev
 529  *   Pointer to the Ethernet device structure.
 530  * @param[in] flow
 531  *   Pointer to flow structure.
 532  */
 533 static void
 534 mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 535 {
 536         struct priv *priv = dev->data->dev_private;
 537         const int mark = !!(flow->actions &
 538                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 539         const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 540         unsigned int i;
 541
 542         for (i = 0; i != flow->rss.queue_num; ++i) {
 543                 int idx = (*flow->queue)[i];
 544                 struct mlx5_rxq_ctrl *rxq_ctrl =
 545                         container_of((*priv->rxqs)[idx],
 546                                      struct mlx5_rxq_ctrl, rxq);
 547
 548                 if (mark) {
 549                         rxq_ctrl->rxq.mark = 1;
 550                         rxq_ctrl->flow_mark_n++;
 551                 }
 552                 if (tunnel) {
 553                         unsigned int j;
 554
 555                         /* Increase the counter matching the flow. */
 556                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 557                                 if ((tunnels_info[j].tunnel & flow->layers) ==
 558                                     tunnels_info[j].tunnel) {
 559                                         rxq_ctrl->flow_tunnels_n[j]++;
 560                                         break;
 561                                 }
 562                         }
 563                         mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 564                 }
 565         }
 566 }
 567
 568 /**
 569  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
 570  * @p flow if no other flow uses it with the same kind of request.
 571  *
 572  * @param dev
 573  *   Pointer to Ethernet device.
 574  * @param[in] flow
 575  *   Pointer to the flow.
 576  */
 577 static void
 578 mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 579 {
 580         struct priv *priv = dev->data->dev_private;
 581         const int mark = !!(flow->actions &
 582                             (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
 583         const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 584         unsigned int i;
 585
 586         assert(dev->data->dev_started);
 587         for (i = 0; i != flow->rss.queue_num; ++i) {
 588                 int idx = (*flow->queue)[i];
 589                 struct mlx5_rxq_ctrl *rxq_ctrl =
 590                         container_of((*priv->rxqs)[idx],
 591                                      struct mlx5_rxq_ctrl, rxq);
 592
 593                 if (mark) {
 594                         rxq_ctrl->flow_mark_n--;
 595                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 596                 }
 597                 if (tunnel) {
 598                         unsigned int j;
 599
 600                         /* Decrease the counter matching the flow. */
 601                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
 602                                 if ((tunnels_info[j].tunnel & flow->layers) ==
 603                                     tunnels_info[j].tunnel) {
 604                                         rxq_ctrl->flow_tunnels_n[j]--;
 605                                         break;
 606                                 }
 607                         }
 608                         mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 609                 }
 610         }
 611 }
 612
 613 /**
 614  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
 615  *
 616  * @param dev
 617  *   Pointer to Ethernet device.
 618  */
 619 static void
 620 mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 621 {
 622         struct priv *priv = dev->data->dev_private;
 623         unsigned int i;
 624
 625         for (i = 0; i != priv->rxqs_n; ++i) {
 626                 struct mlx5_rxq_ctrl *rxq_ctrl;
 627                 unsigned int j;
 628
 629                 if (!(*priv->rxqs)[i])
 630                         continue;
 631                 rxq_ctrl = container_of((*priv->rxqs)[i],
 632                                         struct mlx5_rxq_ctrl, rxq);
 633                 rxq_ctrl->flow_mark_n = 0;
 634                 rxq_ctrl->rxq.mark = 0;
 635                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
 636                         rxq_ctrl->flow_tunnels_n[j] = 0;
 637                 rxq_ctrl->rxq.tunnel = 0;
 638         }
 639 }
 640
 641 /*
 642  * Validate the flag action.
 643  *
 644  * @param[in] action_flags
 645  *   Bit-fields that holds the actions detected until now.
 646  * @param[out] error
 647  *   Pointer to error structure.
 648  *
 649  * @return
 650  *   0 on success, a negative errno value otherwise and rte_errno is set.
 651  */
 652 int
 653 mlx5_flow_validate_action_flag(uint64_t action_flags,
 654                                struct rte_flow_error *error)
 655 {
 656
 657         if (action_flags & MLX5_FLOW_ACTION_DROP)
 658                 return rte_flow_error_set(error, EINVAL,
 659                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 660                                           "can't drop and flag in same flow");
 661         if (action_flags & MLX5_FLOW_ACTION_MARK)
 662                 return rte_flow_error_set(error, EINVAL,
 663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 664                                           "can't mark and flag in same flow");
 665         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 666                 return rte_flow_error_set(error, EINVAL,
 667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 668                                           "can't have 2 flag"
 669                                           " actions in same flow");
 670         return 0;
 671 }
 672
 673 /*
 674  * Validate the mark action.
 675  *
 676  * @param[in] action
 677  *   Pointer to the queue action.
 678  * @param[in] action_flags
 679  *   Bit-fields that holds the actions detected until now.
 680  * @param[out] error
 681  *   Pointer to error structure.
 682  *
 683  * @return
 684  *   0 on success, a negative errno value otherwise and rte_errno is set.
 685  */
 686 int
 687 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
 688                                uint64_t action_flags,
 689                                struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_mark *mark = action->conf;
 692
 693         if (!mark)
 694                 return rte_flow_error_set(error, EINVAL,
 695                                           RTE_FLOW_ERROR_TYPE_ACTION,
 696                                           action,
 697                                           "configuration cannot be null");
 698         if (mark->id >= MLX5_FLOW_MARK_MAX)
 699                 return rte_flow_error_set(error, EINVAL,
 700                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 701                                           &mark->id,
 702                                           "mark id must in 0 <= id < "
 703                                           RTE_STR(MLX5_FLOW_MARK_MAX));
 704         if (action_flags & MLX5_FLOW_ACTION_DROP)
 705                 return rte_flow_error_set(error, EINVAL,
 706                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 707                                           "can't drop and mark in same flow");
 708         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 709                 return rte_flow_error_set(error, EINVAL,
 710                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 711                                           "can't flag and mark in same flow");
 712         if (action_flags & MLX5_FLOW_ACTION_MARK)
 713                 return rte_flow_error_set(error, EINVAL,
 714                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 715                                           "can't have 2 mark actions in same"
 716                                           " flow");
 717         return 0;
 718 }
 719
 720 /*
 721  * Validate the drop action.
 722  *
 723  * @param[in] action_flags
 724  *   Bit-fields that holds the actions detected until now.
 725  * @param[out] error
 726  *   Pointer to error structure.
 727  *
 728  * @return
 729  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 730  */
 731 int
 732 mlx5_flow_validate_action_drop(uint64_t action_flags,
 733                                struct rte_flow_error *error)
 734 {
 735         if (action_flags & MLX5_FLOW_ACTION_FLAG)
 736                 return rte_flow_error_set(error, EINVAL,
 737                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 738                                           "can't drop and flag in same flow");
 739         if (action_flags & MLX5_FLOW_ACTION_MARK)
 740                 return rte_flow_error_set(error, EINVAL,
 741                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 742                                           "can't drop and mark in same flow");
 743         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 744                 return rte_flow_error_set(error, EINVAL,
 745                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 746                                           "can't have 2 fate actions in"
 747                                           " same flow");
 748         return 0;
 749 }
 750
 751 /*
 752  * Validate the queue action.
 753  *
 754  * @param[in] action
 755  *   Pointer to the queue action.
 756  * @param[in] action_flags
 757  *   Bit-fields that holds the actions detected until now.
 758  * @param[in] dev
 759  *   Pointer to the Ethernet device structure.
 760  * @param[out] error
 761  *   Pointer to error structure.
 762  *
 763  * @return
 764  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 765  */
 766 int
 767 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
 768                                 uint64_t action_flags,
 769                                 struct rte_eth_dev *dev,
 770                                 struct rte_flow_error *error)
 771 {
 772         struct priv *priv = dev->data->dev_private;
 773         const struct rte_flow_action_queue *queue = action->conf;
 774
 775         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 776                 return rte_flow_error_set(error, EINVAL,
 777                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 778                                           "can't have 2 fate actions in"
 779                                           " same flow");
 780         if (queue->index >= priv->rxqs_n)
 781                 return rte_flow_error_set(error, EINVAL,
 782                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 783                                           &queue->index,
 784                                           "queue index out of range");
 785         if (!(*priv->rxqs)[queue->index])
 786                 return rte_flow_error_set(error, EINVAL,
 787                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 788                                           &queue->index,
 789                                           "queue is not configured");
 790         return 0;
 791 }
 792
 793 /*
 794  * Validate the rss action.
 795  *
 796  * @param[in] action
 797  *   Pointer to the queue action.
 798  * @param[in] action_flags
 799  *   Bit-fields that holds the actions detected until now.
 800  * @param[in] dev
 801  *   Pointer to the Ethernet device structure.
 802  * @param[out] error
 803  *   Pointer to error structure.
 804  *
 805  * @return
 806  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 807  */
 808 int
 809 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
 810                               uint64_t action_flags,
 811                               struct rte_eth_dev *dev,
 812                               struct rte_flow_error *error)
 813 {
 814         struct priv *priv = dev->data->dev_private;
 815         const struct rte_flow_action_rss *rss = action->conf;
 816         unsigned int i;
 817
 818         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
 819                 return rte_flow_error_set(error, EINVAL,
 820                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 821                                           "can't have 2 fate actions"
 822                                           " in same flow");
 823         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
 824             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 825                 return rte_flow_error_set(error, ENOTSUP,
 826                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 827                                           &rss->func,
 828                                           "RSS hash function not supported");
 829 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 830         if (rss->level > 2)
 831 #else
 832         if (rss->level > 1)
 833 #endif
 834                 return rte_flow_error_set(error, ENOTSUP,
 835                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 836                                           &rss->level,
 837                                           "tunnel RSS is not supported");
 838         if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
 839                 return rte_flow_error_set(error, ENOTSUP,
 840                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 841                                           &rss->key_len,
 842                                           "RSS hash key too small");
 843         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
 844                 return rte_flow_error_set(error, ENOTSUP,
 845                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 846                                           &rss->key_len,
 847                                           "RSS hash key too large");
 848         if (rss->queue_num > priv->config.ind_table_max_size)
 849                 return rte_flow_error_set(error, ENOTSUP,
 850                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 851                                           &rss->queue_num,
 852                                           "number of queues too large");
 853         if (rss->types & MLX5_RSS_HF_MASK)
 854                 return rte_flow_error_set(error, ENOTSUP,
 855                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 856                                           &rss->types,
 857                                           "some RSS protocols are not"
 858                                           " supported");
 859         for (i = 0; i != rss->queue_num; ++i) {
 860                 if (!(*priv->rxqs)[rss->queue[i]])
 861                         return rte_flow_error_set
 862                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 863                                  &rss->queue[i], "queue is not configured");
 864         }
 865         return 0;
 866 }
 867
 868 /*
 869  * Validate the count action.
 870  *
 871  * @param[in] dev
 872  *   Pointer to the Ethernet device structure.
 873  * @param[out] error
 874  *   Pointer to error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 878  */
 879 int
 880 mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
 881                                 struct rte_flow_error *error)
 882 {
 883         struct priv *priv = dev->data->dev_private;
 884
 885         if (!priv->config.flow_counter_en)
 886                 return rte_flow_error_set(error, ENOTSUP,
 887                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 888                                           "flow counters are not supported.");
 889         return 0;
 890 }
 891
 892 /**
 893  * Verify the @p attributes will be correctly understood by the NIC and store
 894  * them in the @p flow if everything is correct.
 895  *
 896  * @param[in] dev
 897  *   Pointer to the Ethernet device structure.
 898  * @param[in] attributes
 899  *   Pointer to flow attributes
 900  * @param[out] error
 901  *   Pointer to error structure.
 902  *
 903  * @return
 904  *   0 on success, a negative errno value otherwise and rte_errno is set.
 905  */
 906 int
 907 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
 908                               const struct rte_flow_attr *attributes,
 909                               struct rte_flow_error *error)
 910 {
 911         struct priv *priv = dev->data->dev_private;
 912         uint32_t priority_max = priv->config.flow_prio - 1;
 913
 914         if (attributes->group)
 915                 return rte_flow_error_set(error, ENOTSUP,
 916                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 917                                           NULL, "groups is not supported");
 918         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 919             attributes->priority >= priority_max)
 920                 return rte_flow_error_set(error, ENOTSUP,
 921                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 922                                           NULL, "priority out of range");
 923         if (attributes->egress)
 924                 return rte_flow_error_set(error, ENOTSUP,
 925                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 926                                           "egress is not supported");
 927         if (attributes->transfer)
 928                 return rte_flow_error_set(error, ENOTSUP,
 929                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 930                                           NULL, "transfer is not supported");
 931         if (!attributes->ingress)
 932                 return rte_flow_error_set(error, EINVAL,
 933                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 934                                           NULL,
 935                                           "ingress attribute is mandatory");
 936         return 0;
 937 }
 938
 939 /**
 940  * Validate Ethernet item.
 941  *
 942  * @param[in] item
 943  *   Item specification.
 944  * @param[in] item_flags
 945  *   Bit-fields that holds the items detected until now.
 946  * @param[out] error
 947  *   Pointer to error structure.
 948  *
 949  * @return
 950  *   0 on success, a negative errno value otherwise and rte_errno is set.
 951  */
 952 int
 953 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
 954                             uint64_t item_flags,
 955                             struct rte_flow_error *error)
 956 {
 957         const struct rte_flow_item_eth *mask = item->mask;
 958         const struct rte_flow_item_eth nic_mask = {
 959                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 960                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 961                 .type = RTE_BE16(0xffff),
 962         };
 963         int ret;
 964         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 965
 966         if (item_flags & MLX5_FLOW_LAYER_OUTER_L2)
 967                 return rte_flow_error_set(error, ENOTSUP,
 968                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 969                                           "3 levels of l2 are not supported");
 970         if ((item_flags & MLX5_FLOW_LAYER_INNER_L2) && !tunnel)
 971                 return rte_flow_error_set(error, ENOTSUP,
 972                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 973                                           "2 L2 without tunnel are not supported");
 974         if (!mask)
 975                 mask = &rte_flow_item_eth_mask;
 976         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 977                                         (const uint8_t *)&nic_mask,
 978                                         sizeof(struct rte_flow_item_eth),
 979                                         error);
 980         return ret;
 981 }
 982
 983 /**
 984  * Validate VLAN item.
 985  *
 986  * @param[in] item
 987  *   Item specification.
 988  * @param[in] item_flags
 989  *   Bit-fields that holds the items detected until now.
 990  * @param[out] error
 991  *   Pointer to error structure.
 992  *
 993  * @return
 994  *   0 on success, a negative errno value otherwise and rte_errno is set.
 995  */
 996 int
 997 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
 998                              int64_t item_flags,
 999                              struct rte_flow_error *error)
1000 {
1001         const struct rte_flow_item_vlan *spec = item->spec;
1002         const struct rte_flow_item_vlan *mask = item->mask;
1003         const struct rte_flow_item_vlan nic_mask = {
1004                 .tci = RTE_BE16(0x0fff),
1005                 .inner_type = RTE_BE16(0xffff),
1006         };
1007         uint16_t vlan_tag = 0;
1008         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1009         int ret;
1010         const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1011                                         MLX5_FLOW_LAYER_INNER_L4) :
1012                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1013                                         MLX5_FLOW_LAYER_OUTER_L4);
1014         const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1015                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1016
1017         if (item_flags & vlanm)
1018                 return rte_flow_error_set(error, EINVAL,
1019                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1020                                           "VLAN layer already configured");
1021         else if ((item_flags & l34m) != 0)
1022                 return rte_flow_error_set(error, EINVAL,
1023                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1024                                           "L2 layer cannot follow L3/L4 layer");
1025         if (!mask)
1026                 mask = &rte_flow_item_vlan_mask;
1027         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1028                                         (const uint8_t *)&nic_mask,
1029                                         sizeof(struct rte_flow_item_vlan),
1030                                         error);
1031         if (ret)
1032                 return ret;
1033         if (spec) {
1034                 vlan_tag = spec->tci;
1035                 vlan_tag &= mask->tci;
1036         }
1037         /*
1038          * From verbs perspective an empty VLAN is equivalent
1039          * to a packet without VLAN layer.
1040          */
1041         if (!vlan_tag)
1042                 return rte_flow_error_set(error, EINVAL,
1043                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1044                                           item->spec,
1045                                           "VLAN cannot be empty");
1046         return 0;
1047 }
1048
1049 /**
1050  * Validate IPV4 item.
1051  *
1052  * @param[in] item
1053  *   Item specification.
1054  * @param[in] item_flags
1055  *   Bit-fields that holds the items detected until now.
1056  * @param[out] error
1057  *   Pointer to error structure.
1058  *
1059  * @return
1060  *   0 on success, a negative errno value otherwise and rte_errno is set.
1061  */
1062 int
1063 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1064                              int64_t item_flags,
1065                              struct rte_flow_error *error)
1066 {
1067         const struct rte_flow_item_ipv4 *mask = item->mask;
1068         const struct rte_flow_item_ipv4 nic_mask = {
1069                 .hdr = {
1070                         .src_addr = RTE_BE32(0xffffffff),
1071                         .dst_addr = RTE_BE32(0xffffffff),
1072                         .type_of_service = 0xff,
1073                         .next_proto_id = 0xff,
1074                 },
1075         };
1076         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1077         int ret;
1078
1079         if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1080                                    MLX5_FLOW_LAYER_OUTER_L3))
1081                 return rte_flow_error_set(error, ENOTSUP,
1082                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1083                                           "multiple L3 layers not supported");
1084         else if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1085                                         MLX5_FLOW_LAYER_OUTER_L4))
1086                 return rte_flow_error_set(error, EINVAL,
1087                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1088                                           "L3 cannot follow an L4 layer.");
1089         if (!mask)
1090                 mask = &rte_flow_item_ipv4_mask;
1091         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1092                                         (const uint8_t *)&nic_mask,
1093                                         sizeof(struct rte_flow_item_ipv4),
1094                                         error);
1095         if (ret < 0)
1096                 return ret;
1097         return 0;
1098 }
1099
1100 /**
1101  * Validate IPV6 item.
1102  *
1103  * @param[in] item
1104  *   Item specification.
1105  * @param[in] item_flags
1106  *   Bit-fields that holds the items detected until now.
1107  * @param[out] error
1108  *   Pointer to error structure.
1109  *
1110  * @return
1111  *   0 on success, a negative errno value otherwise and rte_errno is set.
1112  */
1113 int
1114 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1115                              uint64_t item_flags,
1116                              struct rte_flow_error *error)
1117 {
1118         const struct rte_flow_item_ipv6 *mask = item->mask;
1119         const struct rte_flow_item_ipv6 nic_mask = {
1120                 .hdr = {
1121                         .src_addr =
1122                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1123                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1124                         .dst_addr =
1125                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1126                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1127                         .vtc_flow = RTE_BE32(0xffffffff),
1128                         .proto = 0xff,
1129                         .hop_limits = 0xff,
1130                 },
1131         };
1132         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1133         int ret;
1134
1135         if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1136                                    MLX5_FLOW_LAYER_OUTER_L3))
1137                 return rte_flow_error_set(error, ENOTSUP,
1138                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1139                                           "multiple L3 layers not supported");
1140         else if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1141                                         MLX5_FLOW_LAYER_OUTER_L4))
1142                 return rte_flow_error_set(error, EINVAL,
1143                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1144                                           "L3 cannot follow an L4 layer.");
1145         /*
1146          * IPv6 is not recognised by the NIC inside a GRE tunnel.
1147          * Such support has to be disabled as the rule will be
1148          * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
1149          * Mellanox OFED 4.4-1.0.0.0.
1150          */
1151         if (tunnel && item_flags & MLX5_FLOW_LAYER_GRE)
1152                 return rte_flow_error_set(error, ENOTSUP,
1153                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1154                                           "IPv6 inside a GRE tunnel is"
1155                                           " not recognised.");
1156         if (!mask)
1157                 mask = &rte_flow_item_ipv6_mask;
1158         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1159                                         (const uint8_t *)&nic_mask,
1160                                         sizeof(struct rte_flow_item_ipv6),
1161                                         error);
1162         if (ret < 0)
1163                 return ret;
1164         return 0;
1165 }
1166
1167 /**
1168  * Validate UDP item.
1169  *
1170  * @param[in] item
1171  *   Item specification.
1172  * @param[in] item_flags
1173  *   Bit-fields that holds the items detected until now.
1174  * @param[in] target_protocol
1175  *   The next protocol in the previous item.
1176  * @param[out] error
1177  *   Pointer to error structure.
1178  *
1179  * @return
1180  *   0 on success, a negative errno value otherwise and rte_errno is set.
1181  */
1182 int
1183 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1184                             uint64_t item_flags,
1185                             uint8_t target_protocol,
1186                             struct rte_flow_error *error)
1187 {
1188         const struct rte_flow_item_udp *mask = item->mask;
1189         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1190         int ret;
1191
1192         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1193                 return rte_flow_error_set(error, EINVAL,
1194                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1195                                           "protocol filtering not compatible"
1196                                           " with UDP layer");
1197         if (!(item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1198                                      MLX5_FLOW_LAYER_OUTER_L3)))
1199                 return rte_flow_error_set(error, EINVAL,
1200                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1201                                           "L3 is mandatory to filter on L4");
1202         if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1203                                    MLX5_FLOW_LAYER_OUTER_L4))
1204                 return rte_flow_error_set(error, EINVAL,
1205                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1206                                           "L4 layer is already present");
1207         if (!mask)
1208                 mask = &rte_flow_item_udp_mask;
1209         ret = mlx5_flow_item_acceptable
1210                 (item, (const uint8_t *)mask,
1211                  (const uint8_t *)&rte_flow_item_udp_mask,
1212                  sizeof(struct rte_flow_item_udp), error);
1213         if (ret < 0)
1214                 return ret;
1215         return 0;
1216 }
1217
1218 /**
1219  * Validate TCP item.
1220  *
1221  * @param[in] item
1222  *   Item specification.
1223  * @param[in] item_flags
1224  *   Bit-fields that holds the items detected until now.
1225  * @param[in] target_protocol
1226  *   The next protocol in the previous item.
1227  * @param[out] error
1228  *   Pointer to error structure.
1229  *
1230  * @return
1231  *   0 on success, a negative errno value otherwise and rte_errno is set.
1232  */
1233 int
1234 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1235                             uint64_t item_flags,
1236                             uint8_t target_protocol,
1237                             struct rte_flow_error *error)
1238 {
1239         const struct rte_flow_item_tcp *mask = item->mask;
1240         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1241         int ret;
1242
1243         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1244                 return rte_flow_error_set(error, EINVAL,
1245                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1246                                           "protocol filtering not compatible"
1247                                           " with TCP layer");
1248         if (!(item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1249                                      MLX5_FLOW_LAYER_OUTER_L3)))
1250                 return rte_flow_error_set(error, EINVAL,
1251                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1252                                           "L3 is mandatory to filter on L4");
1253         if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1254                                    MLX5_FLOW_LAYER_OUTER_L4))
1255                 return rte_flow_error_set(error, EINVAL,
1256                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1257                                           "L4 layer is already present");
1258         if (!mask)
1259                 mask = &rte_flow_item_tcp_mask;
1260         ret = mlx5_flow_item_acceptable
1261                 (item, (const uint8_t *)mask,
1262                  (const uint8_t *)&rte_flow_item_tcp_mask,
1263                  sizeof(struct rte_flow_item_tcp), error);
1264         if (ret < 0)
1265                 return ret;
1266         return 0;
1267 }
1268
1269 /**
1270  * Validate VXLAN item.
1271  *
1272  * @param[in] item
1273  *   Item specification.
1274  * @param[in] item_flags
1275  *   Bit-fields that holds the items detected until now.
1276  * @param[in] target_protocol
1277  *   The next protocol in the previous item.
1278  * @param[out] error
1279  *   Pointer to error structure.
1280  *
1281  * @return
1282  *   0 on success, a negative errno value otherwise and rte_errno is set.
1283  */
1284 int
1285 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1286                               uint64_t item_flags,
1287                               struct rte_flow_error *error)
1288 {
1289         const struct rte_flow_item_vxlan *spec = item->spec;
1290         const struct rte_flow_item_vxlan *mask = item->mask;
1291         int ret;
1292         union vni {
1293                 uint32_t vlan_id;
1294                 uint8_t vni[4];
1295         } id = { .vlan_id = 0, };
1296         uint32_t vlan_id = 0;
1297
1298
1299         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1300                 return rte_flow_error_set(error, ENOTSUP,
1301                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1302                                           "a tunnel is already present");
1303         /*
1304          * Verify only UDPv4 is present as defined in
1305          * https://tools.ietf.org/html/rfc7348
1306          */
1307         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1308                 return rte_flow_error_set(error, EINVAL,
1309                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1310                                           "no outer UDP layer found");
1311         if (!mask)
1312                 mask = &rte_flow_item_vxlan_mask;
1313         ret = mlx5_flow_item_acceptable
1314                 (item, (const uint8_t *)mask,
1315                  (const uint8_t *)&rte_flow_item_vxlan_mask,
1316                  sizeof(struct rte_flow_item_vxlan),
1317                  error);
1318         if (ret < 0)
1319                 return ret;
1320         if (spec) {
1321                 memcpy(&id.vni[1], spec->vni, 3);
1322                 vlan_id = id.vlan_id;
1323                 memcpy(&id.vni[1], mask->vni, 3);
1324                 vlan_id &= id.vlan_id;
1325         }
1326         /*
1327          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
1328          * only this layer is defined in the Verbs specification it is
1329          * interpreted as wildcard and all packets will match this
1330          * rule, if it follows a full stack layer (ex: eth / ipv4 /
1331          * udp), all packets matching the layers before will also
1332          * match this rule.  To avoid such situation, VNI 0 is
1333          * currently refused.
1334          */
1335         if (!vlan_id)
1336                 return rte_flow_error_set(error, ENOTSUP,
1337                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1338                                           "VXLAN vni cannot be 0");
1339         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1340                 return rte_flow_error_set(error, ENOTSUP,
1341                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1342                                           "VXLAN tunnel must be fully defined");
1343         return 0;
1344 }
1345
1346 /**
1347  * Validate VXLAN_GPE item.
1348  *
1349  * @param[in] item
1350  *   Item specification.
1351  * @param[in] item_flags
1352  *   Bit-fields that holds the items detected until now.
1353  * @param[in] priv
1354  *   Pointer to the private data structure.
1355  * @param[in] target_protocol
1356  *   The next protocol in the previous item.
1357  * @param[out] error
1358  *   Pointer to error structure.
1359  *
1360  * @return
1361  *   0 on success, a negative errno value otherwise and rte_errno is set.
1362  */
1363 int
1364 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1365                                   uint64_t item_flags,
1366                                   struct rte_eth_dev *dev,
1367                                   struct rte_flow_error *error)
1368 {
1369         struct priv *priv = dev->data->dev_private;
1370         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1371         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1372         int ret;
1373         union vni {
1374                 uint32_t vlan_id;
1375                 uint8_t vni[4];
1376         } id = { .vlan_id = 0, };
1377         uint32_t vlan_id = 0;
1378
1379         if (!priv->config.l3_vxlan_en)
1380                 return rte_flow_error_set(error, ENOTSUP,
1381                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1382                                           "L3 VXLAN is not enabled by device"
1383                                           " parameter and/or not configured in"
1384                                           " firmware");
1385         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1386                 return rte_flow_error_set(error, ENOTSUP,
1387                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1388                                           "a tunnel is already present");
1389         /*
1390          * Verify only UDPv4 is present as defined in
1391          * https://tools.ietf.org/html/rfc7348
1392          */
1393         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1394                 return rte_flow_error_set(error, EINVAL,
1395                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1396                                           "no outer UDP layer found");
1397         if (!mask)
1398                 mask = &rte_flow_item_vxlan_gpe_mask;
1399         ret = mlx5_flow_item_acceptable
1400                 (item, (const uint8_t *)mask,
1401                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1402                  sizeof(struct rte_flow_item_vxlan_gpe),
1403                  error);
1404         if (ret < 0)
1405                 return ret;
1406         if (spec) {
1407                 if (spec->protocol)
1408                         return rte_flow_error_set(error, ENOTSUP,
1409                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1410                                                   item,
1411                                                   "VxLAN-GPE protocol"
1412                                                   " not supported");
1413                 memcpy(&id.vni[1], spec->vni, 3);
1414                 vlan_id = id.vlan_id;
1415                 memcpy(&id.vni[1], mask->vni, 3);
1416                 vlan_id &= id.vlan_id;
1417         }
1418         /*
1419          * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1420          * layer is defined in the Verbs specification it is interpreted as
1421          * wildcard and all packets will match this rule, if it follows a full
1422          * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1423          * before will also match this rule.  To avoid such situation, VNI 0
1424          * is currently refused.
1425          */
1426         if (!vlan_id)
1427                 return rte_flow_error_set(error, ENOTSUP,
1428                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1429                                           "VXLAN-GPE vni cannot be 0");
1430         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1431                 return rte_flow_error_set(error, ENOTSUP,
1432                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1433                                           "VXLAN-GPE tunnel must be fully"
1434                                           " defined");
1435         return 0;
1436 }
1437
1438 /**
1439  * Validate GRE item.
1440  *
1441  * @param[in] item
1442  *   Item specification.
1443  * @param[in] item_flags
1444  *   Bit flags to mark detected items.
1445  * @param[in] target_protocol
1446  *   The next protocol in the previous item.
1447  * @param[out] error
1448  *   Pointer to error structure.
1449  *
1450  * @return
1451  *   0 on success, a negative errno value otherwise and rte_errno is set.
1452  */
1453 int
1454 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
1455                             uint64_t item_flags,
1456                             uint8_t target_protocol,
1457                             struct rte_flow_error *error)
1458 {
1459         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
1460         const struct rte_flow_item_gre *mask = item->mask;
1461         int ret;
1462
1463         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
1464                 return rte_flow_error_set(error, EINVAL,
1465                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1466                                           "protocol filtering not compatible"
1467                                           " with this GRE layer");
1468         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1469                 return rte_flow_error_set(error, ENOTSUP,
1470                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1471                                           "a tunnel is already present");
1472         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1473                 return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "L3 Layer is missing");
1476         if (!mask)
1477                 mask = &rte_flow_item_gre_mask;
1478         ret = mlx5_flow_item_acceptable
1479                 (item, (const uint8_t *)mask,
1480                  (const uint8_t *)&rte_flow_item_gre_mask,
1481                  sizeof(struct rte_flow_item_gre), error);
1482         if (ret < 0)
1483                 return ret;
1484 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
1485         if (spec && (spec->protocol & mask->protocol))
1486                 return rte_flow_error_set(error, ENOTSUP,
1487                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1488                                           "without MPLS support the"
1489                                           " specification cannot be used for"
1490                                           " filtering");
1491 #endif
1492         return 0;
1493 }
1494
1495 /**
1496  * Validate MPLS item.
1497  *
1498  * @param[in] item
1499  *   Item specification.
1500  * @param[in] item_flags
1501  *   Bit-fields that holds the items detected until now.
1502  * @param[in] target_protocol
1503  *   The next protocol in the previous item.
1504  * @param[out] error
1505  *   Pointer to error structure.
1506  *
1507  * @return
1508  *   0 on success, a negative errno value otherwise and rte_errno is set.
1509  */
1510 int
1511 mlx5_flow_validate_item_mpls(const struct rte_flow_item *item __rte_unused,
1512                              uint64_t item_flags __rte_unused,
1513                              uint8_t target_protocol __rte_unused,
1514                              struct rte_flow_error *error)
1515 {
1516 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1517         const struct rte_flow_item_mpls *mask = item->mask;
1518         int ret;
1519
1520         if (target_protocol != 0xff && target_protocol != IPPROTO_MPLS)
1521                 return rte_flow_error_set(error, EINVAL,
1522                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1523                                           "protocol filtering not compatible"
1524                                           " with MPLS layer");
1525         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1526                 return rte_flow_error_set(error, ENOTSUP,
1527                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1528                                           "a tunnel is already"
1529                                           " present");
1530         if (!mask)
1531                 mask = &rte_flow_item_mpls_mask;
1532         ret = mlx5_flow_item_acceptable
1533                 (item, (const uint8_t *)mask,
1534                  (const uint8_t *)&rte_flow_item_mpls_mask,
1535                  sizeof(struct rte_flow_item_mpls), error);
1536         if (ret < 0)
1537                 return ret;
1538         return 0;
1539 #endif
1540         return rte_flow_error_set(error, ENOTSUP,
1541                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
1542                                   "MPLS is not supported by Verbs, please"
1543                                   " update.");
1544 }
1545
1546 static int
1547 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
1548                    const struct rte_flow_attr *attr __rte_unused,
1549                    const struct rte_flow_item items[] __rte_unused,
1550                    const struct rte_flow_action actions[] __rte_unused,
1551                    struct rte_flow_error *error __rte_unused)
1552 {
1553         rte_errno = ENOTSUP;
1554         return -rte_errno;
1555 }
1556
1557 static struct mlx5_flow *
1558 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
1559                   const struct rte_flow_item items[] __rte_unused,
1560                   const struct rte_flow_action actions[] __rte_unused,
1561                   uint64_t *item_flags __rte_unused,
1562                   uint64_t *action_flags __rte_unused,
1563                   struct rte_flow_error *error __rte_unused)
1564 {
1565         rte_errno = ENOTSUP;
1566         return NULL;
1567 }
1568
1569 static int
1570 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
1571                     struct mlx5_flow *dev_flow __rte_unused,
1572                     const struct rte_flow_attr *attr __rte_unused,
1573                     const struct rte_flow_item items[] __rte_unused,
1574                     const struct rte_flow_action actions[] __rte_unused,
1575                     struct rte_flow_error *error __rte_unused)
1576 {
1577         rte_errno = ENOTSUP;
1578         return -rte_errno;
1579 }
1580
1581 static int
1582 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
1583                 struct rte_flow *flow __rte_unused,
1584                 struct rte_flow_error *error __rte_unused)
1585 {
1586         rte_errno = ENOTSUP;
1587         return -rte_errno;
1588 }
1589
1590 static void
1591 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
1592                  struct rte_flow *flow __rte_unused)
1593 {
1594 }
1595
1596 static void
1597 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
1598                   struct rte_flow *flow __rte_unused)
1599 {
1600 }
1601
1602 /* Void driver to protect from null pointer reference. */
1603 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
1604         .validate = flow_null_validate,
1605         .prepare = flow_null_prepare,
1606         .translate = flow_null_translate,
1607         .apply = flow_null_apply,
1608         .remove = flow_null_remove,
1609         .destroy = flow_null_destroy,
1610 };
1611
1612 /**
1613  * Select flow driver type according to flow attributes and device
1614  * configuration.
1615  *
1616  * @param[in] dev
1617  *   Pointer to the dev structure.
1618  * @param[in] attr
1619  *   Pointer to the flow attributes.
1620  *
1621  * @return
1622  *   flow driver type if supported, MLX5_FLOW_TYPE_MAX otherwise.
1623  */
1624 static enum mlx5_flow_drv_type
1625 flow_get_drv_type(struct rte_eth_dev *dev __rte_unused,
1626                   const struct rte_flow_attr *attr)
1627 {
1628         struct priv *priv __rte_unused = dev->data->dev_private;
1629         enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
1630
1631         if (!attr->transfer) {
1632 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
1633                 type = priv->config.dv_flow_en ?  MLX5_FLOW_TYPE_DV :
1634                                                   MLX5_FLOW_TYPE_VERBS;
1635 #else
1636                 type = MLX5_FLOW_TYPE_VERBS;
1637 #endif
1638         }
1639         return type;
1640 }
1641
1642 #define flow_get_drv_ops(type) flow_drv_ops[type]
1643
1644 /**
1645  * Flow driver validation API. This abstracts calling driver specific functions.
1646  * The type of flow driver is determined according to flow attributes.
1647  *
1648  * @param[in] dev
1649  *   Pointer to the dev structure.
1650  * @param[in] attr
1651  *   Pointer to the flow attributes.
1652  * @param[in] items
1653  *   Pointer to the list of items.
1654  * @param[in] actions
1655  *   Pointer to the list of actions.
1656  * @param[out] error
1657  *   Pointer to the error structure.
1658  *
1659  * @return
1660  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1661  */
1662 static inline int
1663 flow_drv_validate(struct rte_eth_dev *dev,
1664                   const struct rte_flow_attr *attr,
1665                   const struct rte_flow_item items[],
1666                   const struct rte_flow_action actions[],
1667                   struct rte_flow_error *error)
1668 {
1669         const struct mlx5_flow_driver_ops *fops;
1670         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
1671
1672         fops = flow_get_drv_ops(type);
1673         return fops->validate(dev, attr, items, actions, error);
1674 }
1675
1676 /**
1677  * Flow driver preparation API. This abstracts calling driver specific
1678  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1679  * calculates the size of memory required for device flow, allocates the memory,
1680  * initializes the device flow and returns the pointer.
1681  *
1682  * @param[in] attr
1683  *   Pointer to the flow attributes.
1684  * @param[in] items
1685  *   Pointer to the list of items.
1686  * @param[in] actions
1687  *   Pointer to the list of actions.
1688  * @param[out] item_flags
1689  *   Pointer to bit mask of all items detected.
1690  * @param[out] action_flags
1691  *   Pointer to bit mask of all actions detected.
1692  * @param[out] error
1693  *   Pointer to the error structure.
1694  *
1695  * @return
1696  *   Pointer to device flow on success, otherwise NULL and rte_ernno is set.
1697  */
1698 static inline struct mlx5_flow *
1699 flow_drv_prepare(struct rte_flow *flow,
1700                  const struct rte_flow_attr *attr,
1701                  const struct rte_flow_item items[],
1702                  const struct rte_flow_action actions[],
1703                  uint64_t *item_flags,
1704                  uint64_t *action_flags,
1705                  struct rte_flow_error *error)
1706 {
1707         const struct mlx5_flow_driver_ops *fops;
1708         enum mlx5_flow_drv_type type = flow->drv_type;
1709
1710         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1711         fops = flow_get_drv_ops(type);
1712         return fops->prepare(attr, items, actions, item_flags, action_flags,
1713                              error);
1714 }
1715
1716 /**
1717  * Flow driver translation API. This abstracts calling driver specific
1718  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
1719  * translates a generic flow into a driver flow. flow_drv_prepare() must
1720  * precede.
1721  *
1722  *
1723  * @param[in] dev
1724  *   Pointer to the rte dev structure.
1725  * @param[in, out] dev_flow
1726  *   Pointer to the mlx5 flow.
1727  * @param[in] attr
1728  *   Pointer to the flow attributes.
1729  * @param[in] items
1730  *   Pointer to the list of items.
1731  * @param[in] actions
1732  *   Pointer to the list of actions.
1733  * @param[out] error
1734  *   Pointer to the error structure.
1735  *
1736  * @return
1737  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1738  */
1739 static inline int
1740 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
1741                    const struct rte_flow_attr *attr,
1742                    const struct rte_flow_item items[],
1743                    const struct rte_flow_action actions[],
1744                    struct rte_flow_error *error)
1745 {
1746         const struct mlx5_flow_driver_ops *fops;
1747         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
1748
1749         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1750         fops = flow_get_drv_ops(type);
1751         return fops->translate(dev, dev_flow, attr, items, actions, error);
1752 }
1753
1754 /**
1755  * Flow driver apply API. This abstracts calling driver specific functions.
1756  * Parent flow (rte_flow) should have driver type (drv_type). It applies
1757  * translated driver flows on to device. flow_drv_translate() must precede.
1758  *
1759  * @param[in] dev
1760  *   Pointer to Ethernet device structure.
1761  * @param[in, out] flow
1762  *   Pointer to flow structure.
1763  * @param[out] error
1764  *   Pointer to error structure.
1765  *
1766  * @return
1767  *   0 on success, a negative errno value otherwise and rte_errno is set.
1768  */
1769 static inline int
1770 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1771                struct rte_flow_error *error)
1772 {
1773         const struct mlx5_flow_driver_ops *fops;
1774         enum mlx5_flow_drv_type type = flow->drv_type;
1775
1776         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1777         fops = flow_get_drv_ops(type);
1778         return fops->apply(dev, flow, error);
1779 }
1780
1781 /**
1782  * Flow driver remove API. This abstracts calling driver specific functions.
1783  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1784  * on device. All the resources of the flow should be freed by calling
1785  * flow_dv_destroy().
1786  *
1787  * @param[in] dev
1788  *   Pointer to Ethernet device.
1789  * @param[in, out] flow
1790  *   Pointer to flow structure.
1791  */
1792 static inline void
1793 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1794 {
1795         const struct mlx5_flow_driver_ops *fops;
1796         enum mlx5_flow_drv_type type = flow->drv_type;
1797
1798         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1799         fops = flow_get_drv_ops(type);
1800         fops->remove(dev, flow);
1801 }
1802
1803 /**
1804  * Flow driver destroy API. This abstracts calling driver specific functions.
1805  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
1806  * on device and releases resources of the flow.
1807  *
1808  * @param[in] dev
1809  *   Pointer to Ethernet device.
1810  * @param[in, out] flow
1811  *   Pointer to flow structure.
1812  */
1813 static inline void
1814 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1815 {
1816         const struct mlx5_flow_driver_ops *fops;
1817         enum mlx5_flow_drv_type type = flow->drv_type;
1818
1819         assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
1820         fops = flow_get_drv_ops(type);
1821         fops->destroy(dev, flow);
1822 }
1823
1824 /**
1825  * Validate a flow supported by the NIC.
1826  *
1827  * @see rte_flow_validate()
1828  * @see rte_flow_ops
1829  */
1830 int
1831 mlx5_flow_validate(struct rte_eth_dev *dev,
1832                    const struct rte_flow_attr *attr,
1833                    const struct rte_flow_item items[],
1834                    const struct rte_flow_action actions[],
1835                    struct rte_flow_error *error)
1836 {
1837         int ret;
1838
1839         ret = flow_drv_validate(dev, attr, items, actions, error);
1840         if (ret < 0)
1841                 return ret;
1842         return 0;
1843 }
1844
1845 /**
1846  * Get RSS action from the action list.
1847  *
1848  * @param[in] actions
1849  *   Pointer to the list of actions.
1850  *
1851  * @return
1852  *   Pointer to the RSS action if exist, else return NULL.
1853  */
1854 static const struct rte_flow_action_rss*
1855 mlx5_flow_get_rss_action(const struct rte_flow_action actions[])
1856 {
1857         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1858                 switch (actions->type) {
1859                 case RTE_FLOW_ACTION_TYPE_RSS:
1860                         return (const struct rte_flow_action_rss *)
1861                                actions->conf;
1862                 default:
1863                         break;
1864                 }
1865         }
1866         return NULL;
1867 }
1868
1869 static unsigned int
1870 mlx5_find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
1871 {
1872         const struct rte_flow_item *item;
1873         unsigned int has_vlan = 0;
1874
1875         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
1876                 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1877                         has_vlan = 1;
1878                         break;
1879                 }
1880         }
1881         if (has_vlan)
1882                 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
1883                                        MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
1884         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
1885                                MLX5_EXPANSION_ROOT_OUTER;
1886 }
1887
1888 /**
1889  * Create a flow and add it to @p list.
1890  *
1891  * @param dev
1892  *   Pointer to Ethernet device.
1893  * @param list
1894  *   Pointer to a TAILQ flow list.
1895  * @param[in] attr
1896  *   Flow rule attributes.
1897  * @param[in] items
1898  *   Pattern specification (list terminated by the END pattern item).
1899  * @param[in] actions
1900  *   Associated actions (list terminated by the END action).
1901  * @param[out] error
1902  *   Perform verbose error reporting if not NULL.
1903  *
1904  * @return
1905  *   A flow on success, NULL otherwise and rte_errno is set.
1906  */
1907 static struct rte_flow *
1908 mlx5_flow_list_create(struct rte_eth_dev *dev,
1909                       struct mlx5_flows *list,
1910                       const struct rte_flow_attr *attr,
1911                       const struct rte_flow_item items[],
1912                       const struct rte_flow_action actions[],
1913                       struct rte_flow_error *error)
1914 {
1915         struct rte_flow *flow = NULL;
1916         struct mlx5_flow *dev_flow;
1917         uint64_t action_flags = 0;
1918         uint64_t item_flags = 0;
1919         const struct rte_flow_action_rss *rss;
1920         union {
1921                 struct rte_flow_expand_rss buf;
1922                 uint8_t buffer[2048];
1923         } expand_buffer;
1924         struct rte_flow_expand_rss *buf = &expand_buffer.buf;
1925         int ret;
1926         uint32_t i;
1927         uint32_t flow_size;
1928
1929         ret = flow_drv_validate(dev, attr, items, actions, error);
1930         if (ret < 0)
1931                 return NULL;
1932         flow_size = sizeof(struct rte_flow);
1933         rss = mlx5_flow_get_rss_action(actions);
1934         if (rss)
1935                 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
1936                                             sizeof(void *));
1937         else
1938                 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
1939         flow = rte_calloc(__func__, 1, flow_size, 0);
1940         flow->drv_type = flow_get_drv_type(dev, attr);
1941         assert(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
1942                flow->drv_type < MLX5_FLOW_TYPE_MAX);
1943         flow->queue = (void *)(flow + 1);
1944         LIST_INIT(&flow->dev_flows);
1945         if (rss && rss->types) {
1946                 unsigned int graph_root;
1947
1948                 graph_root = mlx5_find_graph_root(items, rss->level);
1949                 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
1950                                           items, rss->types,
1951                                           mlx5_support_expansion,
1952                                           graph_root);
1953                 assert(ret > 0 &&
1954                        (unsigned int)ret < sizeof(expand_buffer.buffer));
1955         } else {
1956                 buf->entries = 1;
1957                 buf->entry[0].pattern = (void *)(uintptr_t)items;
1958         }
1959         for (i = 0; i < buf->entries; ++i) {
1960                 dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern,
1961                                             actions, &item_flags, &action_flags,
1962                                             error);
1963                 if (!dev_flow)
1964                         goto error;
1965                 dev_flow->flow = flow;
1966                 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
1967                 ret = flow_drv_translate(dev, dev_flow, attr,
1968                                          buf->entry[i].pattern,
1969                                          actions, error);
1970                 if (ret < 0)
1971                         goto error;
1972         }
1973         if (dev->data->dev_started) {
1974                 ret = flow_drv_apply(dev, flow, error);
1975                 if (ret < 0)
1976                         goto error;
1977         }
1978         TAILQ_INSERT_TAIL(list, flow, next);
1979         mlx5_flow_rxq_flags_set(dev, flow);
1980         return flow;
1981 error:
1982         ret = rte_errno; /* Save rte_errno before cleanup. */
1983         assert(flow);
1984         flow_drv_destroy(dev, flow);
1985         rte_free(flow);
1986         rte_errno = ret; /* Restore rte_errno. */
1987         return NULL;
1988 }
1989
1990 /**
1991  * Create a flow.
1992  *
1993  * @see rte_flow_create()
1994  * @see rte_flow_ops
1995  */
1996 struct rte_flow *
1997 mlx5_flow_create(struct rte_eth_dev *dev,
1998                  const struct rte_flow_attr *attr,
1999                  const struct rte_flow_item items[],
2000                  const struct rte_flow_action actions[],
2001                  struct rte_flow_error *error)
2002 {
2003         return mlx5_flow_list_create
2004                 (dev, &((struct priv *)dev->data->dev_private)->flows,
2005                  attr, items, actions, error);
2006 }
2007
2008 /**
2009  * Destroy a flow in a list.
2010  *
2011  * @param dev
2012  *   Pointer to Ethernet device.
2013  * @param list
2014  *   Pointer to a TAILQ flow list.
2015  * @param[in] flow
2016  *   Flow to destroy.
2017  */
2018 static void
2019 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
2020                        struct rte_flow *flow)
2021 {
2022         flow_drv_destroy(dev, flow);
2023         TAILQ_REMOVE(list, flow, next);
2024         /*
2025          * Update RX queue flags only if port is started, otherwise it is
2026          * already clean.
2027          */
2028         if (dev->data->dev_started)
2029                 mlx5_flow_rxq_flags_trim(dev, flow);
2030         rte_free(flow);
2031 }
2032
2033 /**
2034  * Destroy all flows.
2035  *
2036  * @param dev
2037  *   Pointer to Ethernet device.
2038  * @param list
2039  *   Pointer to a TAILQ flow list.
2040  */
2041 void
2042 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
2043 {
2044         while (!TAILQ_EMPTY(list)) {
2045                 struct rte_flow *flow;
2046
2047                 flow = TAILQ_FIRST(list);
2048                 mlx5_flow_list_destroy(dev, list, flow);
2049         }
2050 }
2051
2052 /**
2053  * Remove all flows.
2054  *
2055  * @param dev
2056  *   Pointer to Ethernet device.
2057  * @param list
2058  *   Pointer to a TAILQ flow list.
2059  */
2060 void
2061 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
2062 {
2063         struct rte_flow *flow;
2064
2065         TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
2066                 flow_drv_remove(dev, flow);
2067         mlx5_flow_rxq_flags_clear(dev);
2068 }
2069
2070 /**
2071  * Add all flows.
2072  *
2073  * @param dev
2074  *   Pointer to Ethernet device.
2075  * @param list
2076  *   Pointer to a TAILQ flow list.
2077  *
2078  * @return
2079  *   0 on success, a negative errno value otherwise and rte_errno is set.
2080  */
2081 int
2082 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
2083 {
2084         struct rte_flow *flow;
2085         struct rte_flow_error error;
2086         int ret = 0;
2087
2088         TAILQ_FOREACH(flow, list, next) {
2089                 ret = flow_drv_apply(dev, flow, &error);
2090                 if (ret < 0)
2091                         goto error;
2092                 mlx5_flow_rxq_flags_set(dev, flow);
2093         }
2094         return 0;
2095 error:
2096         ret = rte_errno; /* Save rte_errno before cleanup. */
2097         mlx5_flow_stop(dev, list);
2098         rte_errno = ret; /* Restore rte_errno. */
2099         return -rte_errno;
2100 }
2101
2102 /**
2103  * Verify the flow list is empty
2104  *
2105  * @param dev
2106  *  Pointer to Ethernet device.
2107  *
2108  * @return the number of flows not released.
2109  */
2110 int
2111 mlx5_flow_verify(struct rte_eth_dev *dev)
2112 {
2113         struct priv *priv = dev->data->dev_private;
2114         struct rte_flow *flow;
2115         int ret = 0;
2116
2117         TAILQ_FOREACH(flow, &priv->flows, next) {
2118                 DRV_LOG(DEBUG, "port %u flow %p still referenced",
2119                         dev->data->port_id, (void *)flow);
2120                 ++ret;
2121         }
2122         return ret;
2123 }
2124
2125 /**
2126  * Enable a control flow configured from the control plane.
2127  *
2128  * @param dev
2129  *   Pointer to Ethernet device.
2130  * @param eth_spec
2131  *   An Ethernet flow spec to apply.
2132  * @param eth_mask
2133  *   An Ethernet flow mask to apply.
2134  * @param vlan_spec
2135  *   A VLAN flow spec to apply.
2136  * @param vlan_mask
2137  *   A VLAN flow mask to apply.
2138  *
2139  * @return
2140  *   0 on success, a negative errno value otherwise and rte_errno is set.
2141  */
2142 int
2143 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2144                     struct rte_flow_item_eth *eth_spec,
2145                     struct rte_flow_item_eth *eth_mask,
2146                     struct rte_flow_item_vlan *vlan_spec,
2147                     struct rte_flow_item_vlan *vlan_mask)
2148 {
2149         struct priv *priv = dev->data->dev_private;
2150         const struct rte_flow_attr attr = {
2151                 .ingress = 1,
2152                 .priority = MLX5_FLOW_PRIO_RSVD,
2153         };
2154         struct rte_flow_item items[] = {
2155                 {
2156                         .type = RTE_FLOW_ITEM_TYPE_ETH,
2157                         .spec = eth_spec,
2158                         .last = NULL,
2159                         .mask = eth_mask,
2160                 },
2161                 {
2162                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2163                                               RTE_FLOW_ITEM_TYPE_END,
2164                         .spec = vlan_spec,
2165                         .last = NULL,
2166                         .mask = vlan_mask,
2167                 },
2168                 {
2169                         .type = RTE_FLOW_ITEM_TYPE_END,
2170                 },
2171         };
2172         uint16_t queue[priv->reta_idx_n];
2173         struct rte_flow_action_rss action_rss = {
2174                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
2175                 .level = 0,
2176                 .types = priv->rss_conf.rss_hf,
2177                 .key_len = priv->rss_conf.rss_key_len,
2178                 .queue_num = priv->reta_idx_n,
2179                 .key = priv->rss_conf.rss_key,
2180                 .queue = queue,
2181         };
2182         struct rte_flow_action actions[] = {
2183                 {
2184                         .type = RTE_FLOW_ACTION_TYPE_RSS,
2185                         .conf = &action_rss,
2186                 },
2187                 {
2188                         .type = RTE_FLOW_ACTION_TYPE_END,
2189                 },
2190         };
2191         struct rte_flow *flow;
2192         struct rte_flow_error error;
2193         unsigned int i;
2194
2195         if (!priv->reta_idx_n) {
2196                 rte_errno = EINVAL;
2197                 return -rte_errno;
2198         }
2199         for (i = 0; i != priv->reta_idx_n; ++i)
2200                 queue[i] = (*priv->reta_idx)[i];
2201         flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
2202                                      actions, &error);
2203         if (!flow)
2204                 return -rte_errno;
2205         return 0;
2206 }
2207
2208 /**
2209  * Enable a flow control configured from the control plane.
2210  *
2211  * @param dev
2212  *   Pointer to Ethernet device.
2213  * @param eth_spec
2214  *   An Ethernet flow spec to apply.
2215  * @param eth_mask
2216  *   An Ethernet flow mask to apply.
2217  *
2218  * @return
2219  *   0 on success, a negative errno value otherwise and rte_errno is set.
2220  */
2221 int
2222 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2223                struct rte_flow_item_eth *eth_spec,
2224                struct rte_flow_item_eth *eth_mask)
2225 {
2226         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2227 }
2228
2229 /**
2230  * Destroy a flow.
2231  *
2232  * @see rte_flow_destroy()
2233  * @see rte_flow_ops
2234  */
2235 int
2236 mlx5_flow_destroy(struct rte_eth_dev *dev,
2237                   struct rte_flow *flow,
2238                   struct rte_flow_error *error __rte_unused)
2239 {
2240         struct priv *priv = dev->data->dev_private;
2241
2242         mlx5_flow_list_destroy(dev, &priv->flows, flow);
2243         return 0;
2244 }
2245
2246 /**
2247  * Destroy all flows.
2248  *
2249  * @see rte_flow_flush()
2250  * @see rte_flow_ops
2251  */
2252 int
2253 mlx5_flow_flush(struct rte_eth_dev *dev,
2254                 struct rte_flow_error *error __rte_unused)
2255 {
2256         struct priv *priv = dev->data->dev_private;
2257
2258         mlx5_flow_list_flush(dev, &priv->flows);
2259         return 0;
2260 }
2261
2262 /**
2263  * Isolated mode.
2264  *
2265  * @see rte_flow_isolate()
2266  * @see rte_flow_ops
2267  */
2268 int
2269 mlx5_flow_isolate(struct rte_eth_dev *dev,
2270                   int enable,
2271                   struct rte_flow_error *error)
2272 {
2273         struct priv *priv = dev->data->dev_private;
2274
2275         if (dev->data->dev_started) {
2276                 rte_flow_error_set(error, EBUSY,
2277                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2278                                    NULL,
2279                                    "port must be stopped first");
2280                 return -rte_errno;
2281         }
2282         priv->isolated = !!enable;
2283         if (enable)
2284                 dev->dev_ops = &mlx5_dev_ops_isolate;
2285         else
2286                 dev->dev_ops = &mlx5_dev_ops;
2287         return 0;
2288 }
2289
2290 /**
2291  * Query flow counter.
2292  *
2293  * @param flow
2294  *   Pointer to the flow.
2295  *
2296  * @return
2297  *   0 on success, a negative errno value otherwise and rte_errno is set.
2298  */
2299 static int
2300 mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
2301                       void *data __rte_unused,
2302                       struct rte_flow_error *error)
2303 {
2304 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2305         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
2306                 struct rte_flow_query_count *qc = data;
2307                 uint64_t counters[2] = {0, 0};
2308                 struct ibv_query_counter_set_attr query_cs_attr = {
2309                         .cs = flow->counter->cs,
2310                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2311                 };
2312                 struct ibv_counter_set_data query_out = {
2313                         .out = counters,
2314                         .outlen = 2 * sizeof(uint64_t),
2315                 };
2316                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
2317                                                        &query_out);
2318
2319                 if (err)
2320                         return rte_flow_error_set
2321                                 (error, err,
2322                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2323                                  NULL,
2324                                  "cannot read counter");
2325                 qc->hits_set = 1;
2326                 qc->bytes_set = 1;
2327                 qc->hits = counters[0] - flow->counter->hits;
2328                 qc->bytes = counters[1] - flow->counter->bytes;
2329                 if (qc->reset) {
2330                         flow->counter->hits = counters[0];
2331                         flow->counter->bytes = counters[1];
2332                 }
2333                 return 0;
2334         }
2335         return rte_flow_error_set(error, ENOTSUP,
2336                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2337                                   NULL,
2338                                   "flow does not have counter");
2339 #endif
2340         return rte_flow_error_set(error, ENOTSUP,
2341                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2342                                   NULL,
2343                                   "counters are not available");
2344 }
2345
2346 /**
2347  * Query a flows.
2348  *
2349  * @see rte_flow_query()
2350  * @see rte_flow_ops
2351  */
2352 int
2353 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
2354                 struct rte_flow *flow,
2355                 const struct rte_flow_action *actions,
2356                 void *data,
2357                 struct rte_flow_error *error)
2358 {
2359         int ret = 0;
2360
2361         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2362                 switch (actions->type) {
2363                 case RTE_FLOW_ACTION_TYPE_VOID:
2364                         break;
2365                 case RTE_FLOW_ACTION_TYPE_COUNT:
2366                         ret = mlx5_flow_query_count(flow, data, error);
2367                         break;
2368                 default:
2369                         return rte_flow_error_set(error, ENOTSUP,
2370                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2371                                                   actions,
2372                                                   "action not supported");
2373                 }
2374                 if (ret < 0)
2375                         return ret;
2376         }
2377         return 0;
2378 }
2379
2380 /**
2381  * Convert a flow director filter to a generic flow.
2382  *
2383  * @param dev
2384  *   Pointer to Ethernet device.
2385  * @param fdir_filter
2386  *   Flow director filter to add.
2387  * @param attributes
2388  *   Generic flow parameters structure.
2389  *
2390  * @return
2391  *   0 on success, a negative errno value otherwise and rte_errno is set.
2392  */
2393 static int
2394 mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
2395                          const struct rte_eth_fdir_filter *fdir_filter,
2396                          struct mlx5_fdir *attributes)
2397 {
2398         struct priv *priv = dev->data->dev_private;
2399         const struct rte_eth_fdir_input *input = &fdir_filter->input;
2400         const struct rte_eth_fdir_masks *mask =
2401                 &dev->data->dev_conf.fdir_conf.mask;
2402
2403         /* Validate queue number. */
2404         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2405                 DRV_LOG(ERR, "port %u invalid queue number %d",
2406                         dev->data->port_id, fdir_filter->action.rx_queue);
2407                 rte_errno = EINVAL;
2408                 return -rte_errno;
2409         }
2410         attributes->attr.ingress = 1;
2411         attributes->items[0] = (struct rte_flow_item) {
2412                 .type = RTE_FLOW_ITEM_TYPE_ETH,
2413                 .spec = &attributes->l2,
2414                 .mask = &attributes->l2_mask,
2415         };
2416         switch (fdir_filter->action.behavior) {
2417         case RTE_ETH_FDIR_ACCEPT:
2418                 attributes->actions[0] = (struct rte_flow_action){
2419                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2420                         .conf = &attributes->queue,
2421                 };
2422                 break;
2423         case RTE_ETH_FDIR_REJECT:
2424                 attributes->actions[0] = (struct rte_flow_action){
2425                         .type = RTE_FLOW_ACTION_TYPE_DROP,
2426                 };
2427                 break;
2428         default:
2429                 DRV_LOG(ERR, "port %u invalid behavior %d",
2430                         dev->data->port_id,
2431                         fdir_filter->action.behavior);
2432                 rte_errno = ENOTSUP;
2433                 return -rte_errno;
2434         }
2435         attributes->queue.index = fdir_filter->action.rx_queue;
2436         /* Handle L3. */
2437         switch (fdir_filter->input.flow_type) {
2438         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2439         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2440         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2441                 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2442                         .src_addr = input->flow.ip4_flow.src_ip,
2443                         .dst_addr = input->flow.ip4_flow.dst_ip,
2444                         .time_to_live = input->flow.ip4_flow.ttl,
2445                         .type_of_service = input->flow.ip4_flow.tos,
2446                 };
2447                 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
2448                         .src_addr = mask->ipv4_mask.src_ip,
2449                         .dst_addr = mask->ipv4_mask.dst_ip,
2450                         .time_to_live = mask->ipv4_mask.ttl,
2451                         .type_of_service = mask->ipv4_mask.tos,
2452                         .next_proto_id = mask->ipv4_mask.proto,
2453                 };
2454                 attributes->items[1] = (struct rte_flow_item){
2455                         .type = RTE_FLOW_ITEM_TYPE_IPV4,
2456                         .spec = &attributes->l3,
2457                         .mask = &attributes->l3_mask,
2458                 };
2459                 break;
2460         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2461         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2462         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2463                 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2464                         .hop_limits = input->flow.ipv6_flow.hop_limits,
2465                         .proto = input->flow.ipv6_flow.proto,
2466                 };
2467
2468                 memcpy(attributes->l3.ipv6.hdr.src_addr,
2469                        input->flow.ipv6_flow.src_ip,
2470                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2471                 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2472                        input->flow.ipv6_flow.dst_ip,
2473                        RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2474                 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
2475                        mask->ipv6_mask.src_ip,
2476                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2477                 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
2478                        mask->ipv6_mask.dst_ip,
2479                        RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2480                 attributes->items[1] = (struct rte_flow_item){
2481                         .type = RTE_FLOW_ITEM_TYPE_IPV6,
2482                         .spec = &attributes->l3,
2483                         .mask = &attributes->l3_mask,
2484                 };
2485                 break;
2486         default:
2487                 DRV_LOG(ERR, "port %u invalid flow type%d",
2488                         dev->data->port_id, fdir_filter->input.flow_type);
2489                 rte_errno = ENOTSUP;
2490                 return -rte_errno;
2491         }
2492         /* Handle L4. */
2493         switch (fdir_filter->input.flow_type) {
2494         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2495                 attributes->l4.udp.hdr = (struct udp_hdr){
2496                         .src_port = input->flow.udp4_flow.src_port,
2497                         .dst_port = input->flow.udp4_flow.dst_port,
2498                 };
2499                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2500                         .src_port = mask->src_port_mask,
2501                         .dst_port = mask->dst_port_mask,
2502                 };
2503                 attributes->items[2] = (struct rte_flow_item){
2504                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2505                         .spec = &attributes->l4,
2506                         .mask = &attributes->l4_mask,
2507                 };
2508                 break;
2509         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2510                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2511                         .src_port = input->flow.tcp4_flow.src_port,
2512                         .dst_port = input->flow.tcp4_flow.dst_port,
2513                 };
2514                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2515                         .src_port = mask->src_port_mask,
2516                         .dst_port = mask->dst_port_mask,
2517                 };
2518                 attributes->items[2] = (struct rte_flow_item){
2519                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2520                         .spec = &attributes->l4,
2521                         .mask = &attributes->l4_mask,
2522                 };
2523                 break;
2524         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2525                 attributes->l4.udp.hdr = (struct udp_hdr){
2526                         .src_port = input->flow.udp6_flow.src_port,
2527                         .dst_port = input->flow.udp6_flow.dst_port,
2528                 };
2529                 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2530                         .src_port = mask->src_port_mask,
2531                         .dst_port = mask->dst_port_mask,
2532                 };
2533                 attributes->items[2] = (struct rte_flow_item){
2534                         .type = RTE_FLOW_ITEM_TYPE_UDP,
2535                         .spec = &attributes->l4,
2536                         .mask = &attributes->l4_mask,
2537                 };
2538                 break;
2539         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2540                 attributes->l4.tcp.hdr = (struct tcp_hdr){
2541                         .src_port = input->flow.tcp6_flow.src_port,
2542                         .dst_port = input->flow.tcp6_flow.dst_port,
2543                 };
2544                 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2545                         .src_port = mask->src_port_mask,
2546                         .dst_port = mask->dst_port_mask,
2547                 };
2548                 attributes->items[2] = (struct rte_flow_item){
2549                         .type = RTE_FLOW_ITEM_TYPE_TCP,
2550                         .spec = &attributes->l4,
2551                         .mask = &attributes->l4_mask,
2552                 };
2553                 break;
2554         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2555         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2556                 break;
2557         default:
2558                 DRV_LOG(ERR, "port %u invalid flow type%d",
2559                         dev->data->port_id, fdir_filter->input.flow_type);
2560                 rte_errno = ENOTSUP;
2561                 return -rte_errno;
2562         }
2563         return 0;
2564 }
2565
2566 /**
2567  * Add new flow director filter and store it in list.
2568  *
2569  * @param dev
2570  *   Pointer to Ethernet device.
2571  * @param fdir_filter
2572  *   Flow director filter to add.
2573  *
2574  * @return
2575  *   0 on success, a negative errno value otherwise and rte_errno is set.
2576  */
2577 static int
2578 mlx5_fdir_filter_add(struct rte_eth_dev *dev,
2579                      const struct rte_eth_fdir_filter *fdir_filter)
2580 {
2581         struct priv *priv = dev->data->dev_private;
2582         struct mlx5_fdir attributes = {
2583                 .attr.group = 0,
2584                 .l2_mask = {
2585                         .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2586                         .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2587                         .type = 0,
2588                 },
2589         };
2590         struct rte_flow_error error;
2591         struct rte_flow *flow;
2592         int ret;
2593
2594         ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
2595         if (ret)
2596                 return ret;
2597         flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
2598                                      attributes.items, attributes.actions,
2599                                      &error);
2600         if (flow) {
2601                 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id,
2602                         (void *)flow);
2603                 return 0;
2604         }
2605         return -rte_errno;
2606 }
2607
2608 /**
2609  * Delete specific filter.
2610  *
2611  * @param dev
2612  *   Pointer to Ethernet device.
2613  * @param fdir_filter
2614  *   Filter to be deleted.
2615  *
2616  * @return
2617  *   0 on success, a negative errno value otherwise and rte_errno is set.
2618  */
2619 static int
2620 mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
2621                         const struct rte_eth_fdir_filter *fdir_filter
2622                         __rte_unused)
2623 {
2624         rte_errno = ENOTSUP;
2625         return -rte_errno;
2626 }
2627
2628 /**
2629  * Update queue for specific filter.
2630  *
2631  * @param dev
2632  *   Pointer to Ethernet device.
2633  * @param fdir_filter
2634  *   Filter to be updated.
2635  *
2636  * @return
2637  *   0 on success, a negative errno value otherwise and rte_errno is set.
2638  */
2639 static int
2640 mlx5_fdir_filter_update(struct rte_eth_dev *dev,
2641                         const struct rte_eth_fdir_filter *fdir_filter)
2642 {
2643         int ret;
2644
2645         ret = mlx5_fdir_filter_delete(dev, fdir_filter);
2646         if (ret)
2647                 return ret;
2648         return mlx5_fdir_filter_add(dev, fdir_filter);
2649 }
2650
2651 /**
2652  * Flush all filters.
2653  *
2654  * @param dev
2655  *   Pointer to Ethernet device.
2656  */
2657 static void
2658 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
2659 {
2660         struct priv *priv = dev->data->dev_private;
2661
2662         mlx5_flow_list_flush(dev, &priv->flows);
2663 }
2664
2665 /**
2666  * Get flow director information.
2667  *
2668  * @param dev
2669  *   Pointer to Ethernet device.
2670  * @param[out] fdir_info
2671  *   Resulting flow director information.
2672  */
2673 static void
2674 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
2675 {
2676         struct rte_eth_fdir_masks *mask =
2677                 &dev->data->dev_conf.fdir_conf.mask;
2678
2679         fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
2680         fdir_info->guarant_spc = 0;
2681         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2682         fdir_info->max_flexpayload = 0;
2683         fdir_info->flow_types_mask[0] = 0;
2684         fdir_info->flex_payload_unit = 0;
2685         fdir_info->max_flex_payload_segment_num = 0;
2686         fdir_info->flex_payload_limit = 0;
2687         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2688 }
2689
2690 /**
2691  * Deal with flow director operations.
2692  *
2693  * @param dev
2694  *   Pointer to Ethernet device.
2695  * @param filter_op
2696  *   Operation to perform.
2697  * @param arg
2698  *   Pointer to operation-specific structure.
2699  *
2700  * @return
2701  *   0 on success, a negative errno value otherwise and rte_errno is set.
2702  */
2703 static int
2704 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
2705                     void *arg)
2706 {
2707         enum rte_fdir_mode fdir_mode =
2708                 dev->data->dev_conf.fdir_conf.mode;
2709
2710         if (filter_op == RTE_ETH_FILTER_NOP)
2711                 return 0;
2712         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2713             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2714                 DRV_LOG(ERR, "port %u flow director mode %d not supported",
2715                         dev->data->port_id, fdir_mode);
2716                 rte_errno = EINVAL;
2717                 return -rte_errno;
2718         }
2719         switch (filter_op) {
2720         case RTE_ETH_FILTER_ADD:
2721                 return mlx5_fdir_filter_add(dev, arg);
2722         case RTE_ETH_FILTER_UPDATE:
2723                 return mlx5_fdir_filter_update(dev, arg);
2724         case RTE_ETH_FILTER_DELETE:
2725                 return mlx5_fdir_filter_delete(dev, arg);
2726         case RTE_ETH_FILTER_FLUSH:
2727                 mlx5_fdir_filter_flush(dev);
2728                 break;
2729         case RTE_ETH_FILTER_INFO:
2730                 mlx5_fdir_info_get(dev, arg);
2731                 break;
2732         default:
2733                 DRV_LOG(DEBUG, "port %u unknown operation %u",
2734                         dev->data->port_id, filter_op);
2735                 rte_errno = EINVAL;
2736                 return -rte_errno;
2737         }
2738         return 0;
2739 }
2740
2741 /**
2742  * Manage filter operations.
2743  *
2744  * @param dev
2745  *   Pointer to Ethernet device structure.
2746  * @param filter_type
2747  *   Filter type.
2748  * @param filter_op
2749  *   Operation to perform.
2750  * @param arg
2751  *   Pointer to operation-specific structure.
2752  *
2753  * @return
2754  *   0 on success, a negative errno value otherwise and rte_errno is set.
2755  */
2756 int
2757 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2758                      enum rte_filter_type filter_type,
2759                      enum rte_filter_op filter_op,
2760                      void *arg)
2761 {
2762         switch (filter_type) {
2763         case RTE_ETH_FILTER_GENERIC:
2764                 if (filter_op != RTE_ETH_FILTER_GET) {
2765                         rte_errno = EINVAL;
2766                         return -rte_errno;
2767                 }
2768                 *(const void **)arg = &mlx5_flow_ops;
2769                 return 0;
2770         case RTE_ETH_FILTER_FDIR:
2771                 return mlx5_fdir_ctrl_func(dev, filter_op, arg);
2772         default:
2773                 DRV_LOG(ERR, "port %u filter type (%d) not supported",
2774                         dev->data->port_id, filter_type);
2775                 rte_errno = ENOTSUP;
2776                 return -rte_errno;
2777         }
2778         return 0;
2779 }