drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /* Verbs specification header. */
  43 struct ibv_spec_header {
  44         enum ibv_flow_spec_type type;
  45         uint16_t size;
  46 };
  47
  48 /**
  49  * Discover the maximum number of priority available.
  50  *
  51  * @param[in] dev
  52  *   Pointer to the Ethernet device structure.
  53  *
  54  * @return
  55  *   number of supported flow priority on success, a negative errno
  56  *   value otherwise and rte_errno is set.
  57  */
  58 int
  59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  60 {
  61         struct mlx5_priv *priv = dev->data->dev_private;
  62         struct {
  63                 struct ibv_flow_attr attr;
  64                 struct ibv_flow_spec_eth eth;
  65                 struct ibv_flow_spec_action_drop drop;
  66         } flow_attr = {
  67                 .attr = {
  68                         .num_of_specs = 2,
  69                         .port = (uint8_t)priv->dev_port,
  70                 },
  71                 .eth = {
  72                         .type = IBV_FLOW_SPEC_ETH,
  73                         .size = sizeof(struct ibv_flow_spec_eth),
  74                 },
  75                 .drop = {
  76                         .size = sizeof(struct ibv_flow_spec_action_drop),
  77                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  78                 },
  79         };
  80         struct ibv_flow *flow;
  81         struct mlx5_hrxq *drop = priv->drop_queue.hrxq;
  82         uint16_t vprio[] = { 8, 16 };
  83         int i;
  84         int priority = 0;
  85
  86 #if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35)
  87         /* If DevX supported, driver must support 16 verbs flow priorities. */
  88         priority = RTE_DIM(priority_map_5);
  89         goto out;
  90 #endif
  91         if (!drop->qp) {
  92                 rte_errno = ENOTSUP;
  93                 return -rte_errno;
  94         }
  95         for (i = 0; i != RTE_DIM(vprio); i++) {
  96                 flow_attr.attr.priority = vprio[i] - 1;
  97                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  98                 if (!flow)
  99                         break;
 100                 claim_zero(mlx5_glue->destroy_flow(flow));
 101                 priority = vprio[i];
 102         }
 103         switch (priority) {
 104         case 8:
 105                 priority = RTE_DIM(priority_map_3);
 106                 break;
 107         case 16:
 108                 priority = RTE_DIM(priority_map_5);
 109                 break;
 110         default:
 111                 rte_errno = ENOTSUP;
 112                 DRV_LOG(ERR,
 113                         "port %u verbs maximum priority: %d expected 8/16",
 114                         dev->data->port_id, priority);
 115                 return -rte_errno;
 116         }
 117 #if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35)
 118 out:
 119 #endif
 120         DRV_LOG(INFO, "port %u supported flow priorities:"
 121                 " 0-%d for ingress or egress root table,"
 122                 " 0-%d for non-root table or transfer root table.",
 123                 dev->data->port_id, priority - 2,
 124                 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
 125         return priority;
 126 }
 127
 128 /**
 129  * Adjust flow priority based on the highest layer and the request priority.
 130  *
 131  * @param[in] dev
 132  *   Pointer to the Ethernet device structure.
 133  * @param[in] priority
 134  *   The rule base priority.
 135  * @param[in] subpriority
 136  *   The priority based on the items.
 137  *
 138  * @return
 139  *   The new priority.
 140  */
 141 uint32_t
 142 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 143                                    uint32_t subpriority)
 144 {
 145         uint32_t res = 0;
 146         struct mlx5_priv *priv = dev->data->dev_private;
 147
 148         switch (priv->config.flow_prio) {
 149         case RTE_DIM(priority_map_3):
 150                 res = priority_map_3[priority][subpriority];
 151                 break;
 152         case RTE_DIM(priority_map_5):
 153                 res = priority_map_5[priority][subpriority];
 154                 break;
 155         }
 156         return  res;
 157 }
 158
 159 /**
 160  * Get Verbs flow counter by index.
 161  *
 162  * @param[in] dev
 163  *   Pointer to the Ethernet device structure.
 164  * @param[in] idx
 165  *   mlx5 flow counter index in the container.
 166  * @param[out] ppool
 167  *   mlx5 flow counter pool in the container,
 168  *
 169  * @return
 170  *   A pointer to the counter, NULL otherwise.
 171  */
 172 static struct mlx5_flow_counter *
 173 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 174                               uint32_t idx,
 175                               struct mlx5_flow_counter_pool **ppool)
 176 {
 177         struct mlx5_priv *priv = dev->data->dev_private;
 178         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 179         struct mlx5_flow_counter_pool *pool;
 180
 181         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
 182         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 183         MLX5_ASSERT(pool);
 184         if (ppool)
 185                 *ppool = pool;
 186         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 187 }
 188
 189 /**
 190  * Create Verbs flow counter with Verbs library.
 191  *
 192  * @param[in] dev
 193  *   Pointer to the Ethernet device structure.
 194  * @param[in, out] counter
 195  *   mlx5 flow counter object, contains the counter id,
 196  *   handle of created Verbs flow counter is returned
 197  *   in cs field (if counters are supported).
 198  *
 199  * @return
 200  *   0 On success else a negative errno value is returned
 201  *   and rte_errno is set.
 202  */
 203 static int
 204 flow_verbs_counter_create(struct rte_eth_dev *dev,
 205                           struct mlx5_flow_counter *counter)
 206 {
 207 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 208         struct mlx5_priv *priv = dev->data->dev_private;
 209         struct ibv_context *ctx = priv->sh->cdev->ctx;
 210         struct ibv_counter_set_init_attr init = {
 211                          .counter_set_id = counter->shared_info.id};
 212
 213         counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init);
 214         if (!counter->dcs_when_free) {
 215                 rte_errno = ENOTSUP;
 216                 return -ENOTSUP;
 217         }
 218         return 0;
 219 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 220         struct mlx5_priv *priv = dev->data->dev_private;
 221         struct ibv_context *ctx = priv->sh->cdev->ctx;
 222         struct ibv_counters_init_attr init = {0};
 223         struct ibv_counter_attach_attr attach;
 224         int ret;
 225
 226         memset(&attach, 0, sizeof(attach));
 227         counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init);
 228         if (!counter->dcs_when_free) {
 229                 rte_errno = ENOTSUP;
 230                 return -ENOTSUP;
 231         }
 232         attach.counter_desc = IBV_COUNTER_PACKETS;
 233         attach.index = 0;
 234         ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL);
 235         if (!ret) {
 236                 attach.counter_desc = IBV_COUNTER_BYTES;
 237                 attach.index = 1;
 238                 ret = mlx5_glue->attach_counters
 239                                         (counter->dcs_when_free, &attach, NULL);
 240         }
 241         if (ret) {
 242                 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free));
 243                 counter->dcs_when_free = NULL;
 244                 rte_errno = ret;
 245                 return -ret;
 246         }
 247         return 0;
 248 #else
 249         (void)dev;
 250         (void)counter;
 251         rte_errno = ENOTSUP;
 252         return -ENOTSUP;
 253 #endif
 254 }
 255
 256 /**
 257  * Get a flow counter.
 258  *
 259  * @param[in] dev
 260  *   Pointer to the Ethernet device structure.
 261  * @param[in] id
 262  *   Counter identifier.
 263  *
 264  * @return
 265  *   Index to the counter, 0 otherwise and rte_errno is set.
 266  */
 267 static uint32_t
 268 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t id __rte_unused)
 269 {
 270         struct mlx5_priv *priv = dev->data->dev_private;
 271         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 272         struct mlx5_flow_counter_pool *pool = NULL;
 273         struct mlx5_flow_counter *cnt = NULL;
 274         uint32_t n_valid = cmng->n_valid;
 275         uint32_t pool_idx, cnt_idx;
 276         uint32_t i;
 277         int ret;
 278
 279         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 280                 pool = cmng->pools[pool_idx];
 281                 if (!pool)
 282                         continue;
 283                 cnt = TAILQ_FIRST(&pool->counters[0]);
 284                 if (cnt)
 285                         break;
 286         }
 287         if (!cnt) {
 288                 struct mlx5_flow_counter_pool **pools;
 289                 uint32_t size;
 290
 291                 if (n_valid == cmng->n) {
 292                         /* Resize the container pool array. */
 293                         size = sizeof(struct mlx5_flow_counter_pool *) *
 294                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 295                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 296                                             SOCKET_ID_ANY);
 297                         if (!pools)
 298                                 return 0;
 299                         if (n_valid) {
 300                                 memcpy(pools, cmng->pools,
 301                                        sizeof(struct mlx5_flow_counter_pool *) *
 302                                        n_valid);
 303                                 mlx5_free(cmng->pools);
 304                         }
 305                         cmng->pools = pools;
 306                         cmng->n += MLX5_CNT_CONTAINER_RESIZE;
 307                 }
 308                 /* Allocate memory for new pool*/
 309                 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 310                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 311                 if (!pool)
 312                         return 0;
 313                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 314                         cnt = MLX5_POOL_GET_CNT(pool, i);
 315                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 316                 }
 317                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 318                 cmng->pools[n_valid] = pool;
 319                 pool_idx = n_valid;
 320                 cmng->n_valid++;
 321         }
 322         TAILQ_REMOVE(&pool->counters[0], cnt, next);
 323         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 324         cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
 325         /* Create counter with Verbs. */
 326         ret = flow_verbs_counter_create(dev, cnt);
 327         if (!ret) {
 328                 cnt->dcs_when_active = cnt->dcs_when_free;
 329                 cnt->hits = 0;
 330                 cnt->bytes = 0;
 331                 return cnt_idx;
 332         }
 333         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 334         /* Some error occurred in Verbs library. */
 335         rte_errno = -ret;
 336         return 0;
 337 }
 338
 339 /**
 340  * Release a flow counter.
 341  *
 342  * @param[in] dev
 343  *   Pointer to the Ethernet device structure.
 344  * @param[in] counter
 345  *   Index to the counter handler.
 346  */
 347 static void
 348 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 349 {
 350         struct mlx5_flow_counter_pool *pool;
 351         struct mlx5_flow_counter *cnt;
 352
 353         cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
 354 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 355         claim_zero(mlx5_glue->destroy_counter_set
 356                         ((struct ibv_counter_set *)cnt->dcs_when_active));
 357 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 358         claim_zero(mlx5_glue->destroy_counters
 359                                 ((struct ibv_counters *)cnt->dcs_when_active));
 360 #endif
 361         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 362 }
 363
 364 /**
 365  * Query a flow counter via Verbs library call.
 366  *
 367  * @see rte_flow_query()
 368  * @see rte_flow_ops
 369  */
 370 static int
 371 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 372                          struct rte_flow *flow, void *data,
 373                          struct rte_flow_error *error)
 374 {
 375 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 376         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 377         if (flow->counter) {
 378                 struct mlx5_flow_counter_pool *pool;
 379                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 380                                                 (dev, flow->counter, &pool);
 381                 struct rte_flow_query_count *qc = data;
 382                 uint64_t counters[2] = {0, 0};
 383 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 384                 struct ibv_query_counter_set_attr query_cs_attr = {
 385                         .dcs_when_free = (struct ibv_counter_set *)
 386                                                 cnt->dcs_when_active,
 387                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 388                 };
 389                 struct ibv_counter_set_data query_out = {
 390                         .out = counters,
 391                         .outlen = 2 * sizeof(uint64_t),
 392                 };
 393                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 394                                                        &query_out);
 395 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 396                 int err = mlx5_glue->query_counters
 397                         ((struct ibv_counters *)cnt->dcs_when_active, counters,
 398                                 RTE_DIM(counters),
 399                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 400 #endif
 401                 if (err)
 402                         return rte_flow_error_set
 403                                 (error, err,
 404                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 405                                  NULL,
 406                                  "cannot read counter");
 407                 qc->hits_set = 1;
 408                 qc->bytes_set = 1;
 409                 qc->hits = counters[0] - cnt->hits;
 410                 qc->bytes = counters[1] - cnt->bytes;
 411                 if (qc->reset) {
 412                         cnt->hits = counters[0];
 413                         cnt->bytes = counters[1];
 414                 }
 415                 return 0;
 416         }
 417         return rte_flow_error_set(error, EINVAL,
 418                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 419                                   NULL,
 420                                   "flow does not have counter");
 421 #else
 422         (void)flow;
 423         (void)data;
 424         return rte_flow_error_set(error, ENOTSUP,
 425                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 426                                   NULL,
 427                                   "counters are not available");
 428 #endif
 429 }
 430
 431 /**
 432  * Add a verbs item specification into @p verbs.
 433  *
 434  * @param[out] verbs
 435  *   Pointer to verbs structure.
 436  * @param[in] src
 437  *   Create specification.
 438  * @param[in] size
 439  *   Size in bytes of the specification to copy.
 440  */
 441 static void
 442 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 443                     void *src, unsigned int size)
 444 {
 445         void *dst;
 446
 447         if (!verbs)
 448                 return;
 449         MLX5_ASSERT(verbs->specs);
 450         dst = (void *)(verbs->specs + verbs->size);
 451         memcpy(dst, src, size);
 452         ++verbs->attr.num_of_specs;
 453         verbs->size += size;
 454 }
 455
 456 /**
 457  * Convert the @p item into a Verbs specification. This function assumes that
 458  * the input is valid and that there is space to insert the requested item
 459  * into the flow.
 460  *
 461  * @param[in, out] dev_flow
 462  *   Pointer to dev_flow structure.
 463  * @param[in] item
 464  *   Item specification.
 465  * @param[in] item_flags
 466  *   Parsed item flags.
 467  */
 468 static void
 469 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 470                               const struct rte_flow_item *item,
 471                               uint64_t item_flags)
 472 {
 473         const struct rte_flow_item_eth *spec = item->spec;
 474         const struct rte_flow_item_eth *mask = item->mask;
 475         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 476         struct ibv_flow_spec_eth eth = {
 477                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 478                 .size = size,
 479         };
 480
 481         if (!mask)
 482                 mask = &rte_flow_item_eth_mask;
 483         if (spec) {
 484                 unsigned int i;
 485
 486                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 487                         RTE_ETHER_ADDR_LEN);
 488                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 489                         RTE_ETHER_ADDR_LEN);
 490                 eth.val.ether_type = spec->type;
 491                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 492                         RTE_ETHER_ADDR_LEN);
 493                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 494                         RTE_ETHER_ADDR_LEN);
 495                 eth.mask.ether_type = mask->type;
 496                 /* Remove unwanted bits from values. */
 497                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 498                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 499                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 500                 }
 501                 eth.val.ether_type &= eth.mask.ether_type;
 502         }
 503         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 504 }
 505
 506 /**
 507  * Update the VLAN tag in the Verbs Ethernet specification.
 508  * This function assumes that the input is valid and there is space to add
 509  * the requested item.
 510  *
 511  * @param[in, out] attr
 512  *   Pointer to Verbs attributes structure.
 513  * @param[in] eth
 514  *   Verbs structure containing the VLAN information to copy.
 515  */
 516 static void
 517 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 518                             struct ibv_flow_spec_eth *eth)
 519 {
 520         unsigned int i;
 521         const enum ibv_flow_spec_type search = eth->type;
 522         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 523                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 524
 525         for (i = 0; i != attr->num_of_specs; ++i) {
 526                 if (hdr->type == search) {
 527                         struct ibv_flow_spec_eth *e =
 528                                 (struct ibv_flow_spec_eth *)hdr;
 529
 530                         e->val.vlan_tag = eth->val.vlan_tag;
 531                         e->mask.vlan_tag = eth->mask.vlan_tag;
 532                         e->val.ether_type = eth->val.ether_type;
 533                         e->mask.ether_type = eth->mask.ether_type;
 534                         break;
 535                 }
 536                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 537         }
 538 }
 539
 540 /**
 541  * Convert the @p item into a Verbs specification. This function assumes that
 542  * the input is valid and that there is space to insert the requested item
 543  * into the flow.
 544  *
 545  * @param[in, out] dev_flow
 546  *   Pointer to dev_flow structure.
 547  * @param[in] item
 548  *   Item specification.
 549  * @param[in] item_flags
 550  *   Parsed item flags.
 551  */
 552 static void
 553 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 554                                const struct rte_flow_item *item,
 555                                uint64_t item_flags)
 556 {
 557         const struct rte_flow_item_vlan *spec = item->spec;
 558         const struct rte_flow_item_vlan *mask = item->mask;
 559         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 560         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 561         struct ibv_flow_spec_eth eth = {
 562                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 563                 .size = size,
 564         };
 565         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 566                                       MLX5_FLOW_LAYER_OUTER_L2;
 567
 568         if (!mask)
 569                 mask = &rte_flow_item_vlan_mask;
 570         if (spec) {
 571                 eth.val.vlan_tag = spec->tci;
 572                 eth.mask.vlan_tag = mask->tci;
 573                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 574                 eth.val.ether_type = spec->inner_type;
 575                 eth.mask.ether_type = mask->inner_type;
 576                 eth.val.ether_type &= eth.mask.ether_type;
 577         }
 578         if (!(item_flags & l2m))
 579                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 580         else
 581                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 582         if (!tunnel)
 583                 dev_flow->handle->vf_vlan.tag =
 584                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 585 }
 586
 587 /**
 588  * Convert the @p item into a Verbs specification. This function assumes that
 589  * the input is valid and that there is space to insert the requested item
 590  * into the flow.
 591  *
 592  * @param[in, out] dev_flow
 593  *   Pointer to dev_flow structure.
 594  * @param[in] item
 595  *   Item specification.
 596  * @param[in] item_flags
 597  *   Parsed item flags.
 598  */
 599 static void
 600 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 601                                const struct rte_flow_item *item,
 602                                uint64_t item_flags)
 603 {
 604         const struct rte_flow_item_ipv4 *spec = item->spec;
 605         const struct rte_flow_item_ipv4 *mask = item->mask;
 606         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 607         struct ibv_flow_spec_ipv4_ext ipv4 = {
 608                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 609                 .size = size,
 610         };
 611
 612         if (!mask)
 613                 mask = &rte_flow_item_ipv4_mask;
 614         if (spec) {
 615                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 616                         .src_ip = spec->hdr.src_addr,
 617                         .dst_ip = spec->hdr.dst_addr,
 618                         .proto = spec->hdr.next_proto_id,
 619                         .tos = spec->hdr.type_of_service,
 620                 };
 621                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 622                         .src_ip = mask->hdr.src_addr,
 623                         .dst_ip = mask->hdr.dst_addr,
 624                         .proto = mask->hdr.next_proto_id,
 625                         .tos = mask->hdr.type_of_service,
 626                 };
 627                 /* Remove unwanted bits from values. */
 628                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 629                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 630                 ipv4.val.proto &= ipv4.mask.proto;
 631                 ipv4.val.tos &= ipv4.mask.tos;
 632         }
 633         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 634 }
 635
 636 /**
 637  * Convert the @p item into a Verbs specification. This function assumes that
 638  * the input is valid and that there is space to insert the requested item
 639  * into the flow.
 640  *
 641  * @param[in, out] dev_flow
 642  *   Pointer to dev_flow structure.
 643  * @param[in] item
 644  *   Item specification.
 645  * @param[in] item_flags
 646  *   Parsed item flags.
 647  */
 648 static void
 649 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 650                                const struct rte_flow_item *item,
 651                                uint64_t item_flags)
 652 {
 653         const struct rte_flow_item_ipv6 *spec = item->spec;
 654         const struct rte_flow_item_ipv6 *mask = item->mask;
 655         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 656         struct ibv_flow_spec_ipv6 ipv6 = {
 657                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 658                 .size = size,
 659         };
 660
 661         if (!mask)
 662                 mask = &rte_flow_item_ipv6_mask;
 663         if (spec) {
 664                 unsigned int i;
 665                 uint32_t vtc_flow_val;
 666                 uint32_t vtc_flow_mask;
 667
 668                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 669                        RTE_DIM(ipv6.val.src_ip));
 670                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 671                        RTE_DIM(ipv6.val.dst_ip));
 672                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 673                        RTE_DIM(ipv6.mask.src_ip));
 674                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 675                        RTE_DIM(ipv6.mask.dst_ip));
 676                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 677                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 678                 ipv6.val.flow_label =
 679                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 680                                          RTE_IPV6_HDR_FL_SHIFT);
 681                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 682                                          RTE_IPV6_HDR_TC_SHIFT;
 683                 ipv6.val.next_hdr = spec->hdr.proto;
 684                 ipv6.mask.flow_label =
 685                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 686                                          RTE_IPV6_HDR_FL_SHIFT);
 687                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 688                                           RTE_IPV6_HDR_TC_SHIFT;
 689                 ipv6.mask.next_hdr = mask->hdr.proto;
 690                 /* Remove unwanted bits from values. */
 691                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 692                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 693                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 694                 }
 695                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 696                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 697                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 698         }
 699         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 700 }
 701
 702 /**
 703  * Convert the @p item into a Verbs specification. This function assumes that
 704  * the input is valid and that there is space to insert the requested item
 705  * into the flow.
 706  *
 707  * @param[in, out] dev_flow
 708  *   Pointer to dev_flow structure.
 709  * @param[in] item
 710  *   Item specification.
 711  * @param[in] item_flags
 712  *   Parsed item flags.
 713  */
 714 static void
 715 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 716                               const struct rte_flow_item *item,
 717                               uint64_t item_flags __rte_unused)
 718 {
 719         const struct rte_flow_item_tcp *spec = item->spec;
 720         const struct rte_flow_item_tcp *mask = item->mask;
 721         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 722         struct ibv_flow_spec_tcp_udp tcp = {
 723                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 724                 .size = size,
 725         };
 726
 727         if (!mask)
 728                 mask = &rte_flow_item_tcp_mask;
 729         if (spec) {
 730                 tcp.val.dst_port = spec->hdr.dst_port;
 731                 tcp.val.src_port = spec->hdr.src_port;
 732                 tcp.mask.dst_port = mask->hdr.dst_port;
 733                 tcp.mask.src_port = mask->hdr.src_port;
 734                 /* Remove unwanted bits from values. */
 735                 tcp.val.src_port &= tcp.mask.src_port;
 736                 tcp.val.dst_port &= tcp.mask.dst_port;
 737         }
 738         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 739 }
 740
 741 /**
 742  * Convert the @p item into a Verbs specification. This function assumes that
 743  * the input is valid and that there is space to insert the requested item
 744  * into the flow.
 745  *
 746  * @param[in, out] dev_flow
 747  *   Pointer to dev_flow structure.
 748  * @param[in] item
 749  *   Item specification.
 750  * @param[in] item_flags
 751  *   Parsed item flags.
 752  */
 753 static void
 754 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 755                               const struct rte_flow_item *item,
 756                               uint64_t item_flags __rte_unused)
 757 {
 758         const struct rte_flow_item_udp *spec = item->spec;
 759         const struct rte_flow_item_udp *mask = item->mask;
 760         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 761         struct ibv_flow_spec_tcp_udp udp = {
 762                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 763                 .size = size,
 764         };
 765
 766         if (!mask)
 767                 mask = &rte_flow_item_udp_mask;
 768         if (spec) {
 769                 udp.val.dst_port = spec->hdr.dst_port;
 770                 udp.val.src_port = spec->hdr.src_port;
 771                 udp.mask.dst_port = mask->hdr.dst_port;
 772                 udp.mask.src_port = mask->hdr.src_port;
 773                 /* Remove unwanted bits from values. */
 774                 udp.val.src_port &= udp.mask.src_port;
 775                 udp.val.dst_port &= udp.mask.dst_port;
 776         }
 777         item++;
 778         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 779                 item++;
 780         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 781                 switch ((item)->type) {
 782                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 783                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 784                         udp.mask.dst_port = 0xffff;
 785                         break;
 786                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 787                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 788                         udp.mask.dst_port = 0xffff;
 789                         break;
 790                 case RTE_FLOW_ITEM_TYPE_MPLS:
 791                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 792                         udp.mask.dst_port = 0xffff;
 793                         break;
 794                 default:
 795                         break;
 796                 }
 797         }
 798
 799         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 800 }
 801
 802 /**
 803  * Convert the @p item into a Verbs specification. This function assumes that
 804  * the input is valid and that there is space to insert the requested item
 805  * into the flow.
 806  *
 807  * @param[in, out] dev_flow
 808  *   Pointer to dev_flow structure.
 809  * @param[in] item
 810  *   Item specification.
 811  * @param[in] item_flags
 812  *   Parsed item flags.
 813  */
 814 static void
 815 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 816                                 const struct rte_flow_item *item,
 817                                 uint64_t item_flags __rte_unused)
 818 {
 819         const struct rte_flow_item_vxlan *spec = item->spec;
 820         const struct rte_flow_item_vxlan *mask = item->mask;
 821         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 822         struct ibv_flow_spec_tunnel vxlan = {
 823                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 824                 .size = size,
 825         };
 826         union vni {
 827                 uint32_t vlan_id;
 828                 uint8_t vni[4];
 829         } id = { .vlan_id = 0, };
 830
 831         if (!mask)
 832                 mask = &rte_flow_item_vxlan_mask;
 833         if (spec) {
 834                 memcpy(&id.vni[1], spec->vni, 3);
 835                 vxlan.val.tunnel_id = id.vlan_id;
 836                 memcpy(&id.vni[1], mask->vni, 3);
 837                 vxlan.mask.tunnel_id = id.vlan_id;
 838                 /* Remove unwanted bits from values. */
 839                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 840         }
 841         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 842 }
 843
 844 /**
 845  * Convert the @p item into a Verbs specification. This function assumes that
 846  * the input is valid and that there is space to insert the requested item
 847  * into the flow.
 848  *
 849  * @param[in, out] dev_flow
 850  *   Pointer to dev_flow structure.
 851  * @param[in] item
 852  *   Item specification.
 853  * @param[in] item_flags
 854  *   Parsed item flags.
 855  */
 856 static void
 857 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 858                                     const struct rte_flow_item *item,
 859                                     uint64_t item_flags __rte_unused)
 860 {
 861         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 862         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 863         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 864         struct ibv_flow_spec_tunnel vxlan_gpe = {
 865                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 866                 .size = size,
 867         };
 868         union vni {
 869                 uint32_t vlan_id;
 870                 uint8_t vni[4];
 871         } id = { .vlan_id = 0, };
 872
 873         if (!mask)
 874                 mask = &rte_flow_item_vxlan_gpe_mask;
 875         if (spec) {
 876                 memcpy(&id.vni[1], spec->vni, 3);
 877                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 878                 memcpy(&id.vni[1], mask->vni, 3);
 879                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 880                 /* Remove unwanted bits from values. */
 881                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 882         }
 883         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 884 }
 885
 886 /**
 887  * Update the protocol in Verbs IPv4/IPv6 spec.
 888  *
 889  * @param[in, out] attr
 890  *   Pointer to Verbs attributes structure.
 891  * @param[in] search
 892  *   Specification type to search in order to update the IP protocol.
 893  * @param[in] protocol
 894  *   Protocol value to set if none is present in the specification.
 895  */
 896 static void
 897 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 898                                        enum ibv_flow_spec_type search,
 899                                        uint8_t protocol)
 900 {
 901         unsigned int i;
 902         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 903                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 904
 905         if (!attr)
 906                 return;
 907         for (i = 0; i != attr->num_of_specs; ++i) {
 908                 if (hdr->type == search) {
 909                         union {
 910                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 911                                 struct ibv_flow_spec_ipv6 *ipv6;
 912                         } ip;
 913
 914                         switch (search) {
 915                         case IBV_FLOW_SPEC_IPV4_EXT:
 916                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 917                                 if (!ip.ipv4->val.proto) {
 918                                         ip.ipv4->val.proto = protocol;
 919                                         ip.ipv4->mask.proto = 0xff;
 920                                 }
 921                                 break;
 922                         case IBV_FLOW_SPEC_IPV6:
 923                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 924                                 if (!ip.ipv6->val.next_hdr) {
 925                                         ip.ipv6->val.next_hdr = protocol;
 926                                         ip.ipv6->mask.next_hdr = 0xff;
 927                                 }
 928                                 break;
 929                         default:
 930                                 break;
 931                         }
 932                         break;
 933                 }
 934                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 935         }
 936 }
 937
 938 /**
 939  * Convert the @p item into a Verbs specification. This function assumes that
 940  * the input is valid and that there is space to insert the requested item
 941  * into the flow.
 942  *
 943  * @param[in, out] dev_flow
 944  *   Pointer to dev_flow structure.
 945  * @param[in] item
 946  *   Item specification.
 947  * @param[in] item_flags
 948  *   Parsed item flags.
 949  */
 950 static void
 951 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 952                               const struct rte_flow_item *item __rte_unused,
 953                               uint64_t item_flags)
 954 {
 955         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 956 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 957         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 958         struct ibv_flow_spec_tunnel tunnel = {
 959                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 960                 .size = size,
 961         };
 962 #else
 963         const struct rte_flow_item_gre *spec = item->spec;
 964         const struct rte_flow_item_gre *mask = item->mask;
 965         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 966         struct ibv_flow_spec_gre tunnel = {
 967                 .type = IBV_FLOW_SPEC_GRE,
 968                 .size = size,
 969         };
 970
 971         if (!mask)
 972                 mask = &rte_flow_item_gre_mask;
 973         if (spec) {
 974                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 975                 tunnel.val.protocol = spec->protocol;
 976                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 977                 tunnel.mask.protocol = mask->protocol;
 978                 /* Remove unwanted bits from values. */
 979                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 980                 tunnel.val.protocol &= tunnel.mask.protocol;
 981                 tunnel.val.key &= tunnel.mask.key;
 982         }
 983 #endif
 984         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 985                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 986                                                        IBV_FLOW_SPEC_IPV4_EXT,
 987                                                        IPPROTO_GRE);
 988         else
 989                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 990                                                        IBV_FLOW_SPEC_IPV6,
 991                                                        IPPROTO_GRE);
 992         flow_verbs_spec_add(verbs, &tunnel, size);
 993 }
 994
 995 /**
 996  * Convert the @p action into a Verbs specification. This function assumes that
 997  * the input is valid and that there is space to insert the requested action
 998  * into the flow. This function also return the action that was added.
 999  *
1000  * @param[in, out] dev_flow
1001  *   Pointer to dev_flow structure.
1002  * @param[in] item
1003  *   Item specification.
1004  * @param[in] item_flags
1005  *   Parsed item flags.
1006  */
1007 static void
1008 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1009                                const struct rte_flow_item *item __rte_unused,
1010                                uint64_t item_flags __rte_unused)
1011 {
1012 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1013         const struct rte_flow_item_mpls *spec = item->spec;
1014         const struct rte_flow_item_mpls *mask = item->mask;
1015         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1016         struct ibv_flow_spec_mpls mpls = {
1017                 .type = IBV_FLOW_SPEC_MPLS,
1018                 .size = size,
1019         };
1020
1021         if (!mask)
1022                 mask = &rte_flow_item_mpls_mask;
1023         if (spec) {
1024                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1025                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1026                 /* Remove unwanted bits from values.  */
1027                 mpls.val.label &= mpls.mask.label;
1028         }
1029         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1030 #endif
1031 }
1032
1033 /**
1034  * Convert the @p action into a Verbs specification. This function assumes that
1035  * the input is valid and that there is space to insert the requested action
1036  * into the flow.
1037  *
1038  * @param[in] dev_flow
1039  *   Pointer to mlx5_flow.
1040  * @param[in] action
1041  *   Action configuration.
1042  */
1043 static void
1044 flow_verbs_translate_action_drop
1045         (struct mlx5_flow *dev_flow,
1046          const struct rte_flow_action *action __rte_unused)
1047 {
1048         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1049         struct ibv_flow_spec_action_drop drop = {
1050                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1051                         .size = size,
1052         };
1053
1054         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1055 }
1056
1057 /**
1058  * Convert the @p action into a Verbs specification. This function assumes that
1059  * the input is valid and that there is space to insert the requested action
1060  * into the flow.
1061  *
1062  * @param[in] rss_desc
1063  *   Pointer to mlx5_flow_rss_desc.
1064  * @param[in] action
1065  *   Action configuration.
1066  */
1067 static void
1068 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1069                                   const struct rte_flow_action *action)
1070 {
1071         const struct rte_flow_action_queue *queue = action->conf;
1072
1073         rss_desc->queue[0] = queue->index;
1074         rss_desc->queue_num = 1;
1075 }
1076
1077 /**
1078  * Convert the @p action into a Verbs specification. This function assumes that
1079  * the input is valid and that there is space to insert the requested action
1080  * into the flow.
1081  *
1082  * @param[in] rss_desc
1083  *   Pointer to mlx5_flow_rss_desc.
1084  * @param[in] action
1085  *   Action configuration.
1086  */
1087 static void
1088 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1089                                 const struct rte_flow_action *action)
1090 {
1091         const struct rte_flow_action_rss *rss = action->conf;
1092         const uint8_t *rss_key;
1093
1094         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1095         rss_desc->queue_num = rss->queue_num;
1096         /* NULL RSS key indicates default RSS key. */
1097         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1098         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1099         /*
1100          * rss->level and rss.types should be set in advance when expanding
1101          * items for RSS.
1102          */
1103 }
1104
1105 /**
1106  * Convert the @p action into a Verbs specification. This function assumes that
1107  * the input is valid and that there is space to insert the requested action
1108  * into the flow.
1109  *
1110  * @param[in] dev_flow
1111  *   Pointer to mlx5_flow.
1112  * @param[in] action
1113  *   Action configuration.
1114  */
1115 static void
1116 flow_verbs_translate_action_flag
1117         (struct mlx5_flow *dev_flow,
1118          const struct rte_flow_action *action __rte_unused)
1119 {
1120         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1121         struct ibv_flow_spec_action_tag tag = {
1122                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1123                 .size = size,
1124                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1125         };
1126
1127         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1128 }
1129
1130 /**
1131  * Convert the @p action into a Verbs specification. This function assumes that
1132  * the input is valid and that there is space to insert the requested action
1133  * into the flow.
1134  *
1135  * @param[in] dev_flow
1136  *   Pointer to mlx5_flow.
1137  * @param[in] action
1138  *   Action configuration.
1139  */
1140 static void
1141 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1142                                  const struct rte_flow_action *action)
1143 {
1144         const struct rte_flow_action_mark *mark = action->conf;
1145         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1146         struct ibv_flow_spec_action_tag tag = {
1147                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1148                 .size = size,
1149                 .tag_id = mlx5_flow_mark_set(mark->id),
1150         };
1151
1152         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1153 }
1154
1155 /**
1156  * Convert the @p action into a Verbs specification. This function assumes that
1157  * the input is valid and that there is space to insert the requested action
1158  * into the flow.
1159  *
1160  * @param[in] dev
1161  *   Pointer to the Ethernet device structure.
1162  * @param[in] action
1163  *   Action configuration.
1164  * @param[in] dev_flow
1165  *   Pointer to mlx5_flow.
1166  * @param[out] error
1167  *   Pointer to error structure.
1168  *
1169  * @return
1170  *   0 On success else a negative errno value is returned and rte_errno is set.
1171  */
1172 static int
1173 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1174                                   const struct rte_flow_action *action,
1175                                   struct rte_eth_dev *dev,
1176                                   struct rte_flow_error *error)
1177 {
1178         const struct rte_flow_action_count *count = action->conf;
1179         struct rte_flow *flow = dev_flow->flow;
1180 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1181         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1182         struct mlx5_flow_counter_pool *pool;
1183         struct mlx5_flow_counter *cnt = NULL;
1184         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1185         struct ibv_flow_spec_counter_action counter = {
1186                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1187                 .size = size,
1188         };
1189 #endif
1190
1191         if (!flow->counter) {
1192                 flow->counter = flow_verbs_counter_new(dev, count->id);
1193                 if (!flow->counter)
1194                         return rte_flow_error_set(error, rte_errno,
1195                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1196                                                   action,
1197                                                   "cannot get counter"
1198                                                   " context.");
1199         }
1200 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1201         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1202         counter.counter_set_handle =
1203                 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle;
1204         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1205 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1206         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1207         counter.counters = (struct ibv_counters *)cnt->dcs_when_active;
1208         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1209 #endif
1210         return 0;
1211 }
1212
1213 /**
1214  * Internal validation function. For validating both actions and items.
1215  *
1216  * @param[in] dev
1217  *   Pointer to the Ethernet device structure.
1218  * @param[in] attr
1219  *   Pointer to the flow attributes.
1220  * @param[in] items
1221  *   Pointer to the list of items.
1222  * @param[in] actions
1223  *   Pointer to the list of actions.
1224  * @param[in] external
1225  *   This flow rule is created by request external to PMD.
1226  * @param[in] hairpin
1227  *   Number of hairpin TX actions, 0 means classic flow.
1228  * @param[out] error
1229  *   Pointer to the error structure.
1230  *
1231  * @return
1232  *   0 on success, a negative errno value otherwise and rte_errno is set.
1233  */
1234 static int
1235 flow_verbs_validate(struct rte_eth_dev *dev,
1236                     const struct rte_flow_attr *attr,
1237                     const struct rte_flow_item items[],
1238                     const struct rte_flow_action actions[],
1239                     bool external __rte_unused,
1240                     int hairpin __rte_unused,
1241                     struct rte_flow_error *error)
1242 {
1243         int ret;
1244         uint64_t action_flags = 0;
1245         uint64_t item_flags = 0;
1246         uint64_t last_item = 0;
1247         uint8_t next_protocol = 0xff;
1248         uint16_t ether_type = 0;
1249         bool is_empty_vlan = false;
1250         uint16_t udp_dport = 0;
1251
1252         if (items == NULL)
1253                 return -1;
1254         ret = mlx5_flow_validate_attributes(dev, attr, error);
1255         if (ret < 0)
1256                 return ret;
1257         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1258                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1259                 int ret = 0;
1260
1261                 switch (items->type) {
1262                 case RTE_FLOW_ITEM_TYPE_VOID:
1263                         break;
1264                 case RTE_FLOW_ITEM_TYPE_ETH:
1265                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1266                                                           false, error);
1267                         if (ret < 0)
1268                                 return ret;
1269                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1270                                              MLX5_FLOW_LAYER_OUTER_L2;
1271                         if (items->mask != NULL && items->spec != NULL) {
1272                                 ether_type =
1273                                         ((const struct rte_flow_item_eth *)
1274                                          items->spec)->type;
1275                                 ether_type &=
1276                                         ((const struct rte_flow_item_eth *)
1277                                          items->mask)->type;
1278                                 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN))
1279                                         is_empty_vlan = true;
1280                                 ether_type = rte_be_to_cpu_16(ether_type);
1281                         } else {
1282                                 ether_type = 0;
1283                         }
1284                         break;
1285                 case RTE_FLOW_ITEM_TYPE_VLAN:
1286                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1287                                                            dev, error);
1288                         if (ret < 0)
1289                                 return ret;
1290                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1291                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1292                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1293                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1294                         if (items->mask != NULL && items->spec != NULL) {
1295                                 ether_type =
1296                                         ((const struct rte_flow_item_vlan *)
1297                                          items->spec)->inner_type;
1298                                 ether_type &=
1299                                         ((const struct rte_flow_item_vlan *)
1300                                          items->mask)->inner_type;
1301                                 ether_type = rte_be_to_cpu_16(ether_type);
1302                         } else {
1303                                 ether_type = 0;
1304                         }
1305                         is_empty_vlan = false;
1306                         break;
1307                 case RTE_FLOW_ITEM_TYPE_IPV4:
1308                         ret = mlx5_flow_validate_item_ipv4
1309                                                 (items, item_flags,
1310                                                  last_item, ether_type, NULL,
1311                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1312                                                  error);
1313                         if (ret < 0)
1314                                 return ret;
1315                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1316                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1317                         if (items->mask != NULL &&
1318                             ((const struct rte_flow_item_ipv4 *)
1319                              items->mask)->hdr.next_proto_id) {
1320                                 next_protocol =
1321                                         ((const struct rte_flow_item_ipv4 *)
1322                                          (items->spec))->hdr.next_proto_id;
1323                                 next_protocol &=
1324                                         ((const struct rte_flow_item_ipv4 *)
1325                                          (items->mask))->hdr.next_proto_id;
1326                         } else {
1327                                 /* Reset for inner layer. */
1328                                 next_protocol = 0xff;
1329                         }
1330                         break;
1331                 case RTE_FLOW_ITEM_TYPE_IPV6:
1332                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1333                                                            last_item,
1334                                                            ether_type, NULL,
1335                                                            error);
1336                         if (ret < 0)
1337                                 return ret;
1338                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1339                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1340                         if (items->mask != NULL &&
1341                             ((const struct rte_flow_item_ipv6 *)
1342                              items->mask)->hdr.proto) {
1343                                 next_protocol =
1344                                         ((const struct rte_flow_item_ipv6 *)
1345                                          items->spec)->hdr.proto;
1346                                 next_protocol &=
1347                                         ((const struct rte_flow_item_ipv6 *)
1348                                          items->mask)->hdr.proto;
1349                         } else {
1350                                 /* Reset for inner layer. */
1351                                 next_protocol = 0xff;
1352                         }
1353                         break;
1354                 case RTE_FLOW_ITEM_TYPE_UDP:
1355                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1356                                                           next_protocol,
1357                                                           error);
1358                         const struct rte_flow_item_udp *spec = items->spec;
1359                         const struct rte_flow_item_udp *mask = items->mask;
1360                         if (!mask)
1361                                 mask = &rte_flow_item_udp_mask;
1362                         if (spec != NULL)
1363                                 udp_dport = rte_be_to_cpu_16
1364                                                 (spec->hdr.dst_port &
1365                                                  mask->hdr.dst_port);
1366
1367                         if (ret < 0)
1368                                 return ret;
1369                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1370                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1371                         break;
1372                 case RTE_FLOW_ITEM_TYPE_TCP:
1373                         ret = mlx5_flow_validate_item_tcp
1374                                                 (items, item_flags,
1375                                                  next_protocol,
1376                                                  &rte_flow_item_tcp_mask,
1377                                                  error);
1378                         if (ret < 0)
1379                                 return ret;
1380                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1381                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1382                         break;
1383                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1384                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
1385                                                             items, item_flags,
1386                                                             attr, error);
1387                         if (ret < 0)
1388                                 return ret;
1389                         last_item = MLX5_FLOW_LAYER_VXLAN;
1390                         break;
1391                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1392                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1393                                                                 item_flags,
1394                                                                 dev, error);
1395                         if (ret < 0)
1396                                 return ret;
1397                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1398                         break;
1399                 case RTE_FLOW_ITEM_TYPE_GRE:
1400                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1401                                                           next_protocol, error);
1402                         if (ret < 0)
1403                                 return ret;
1404                         last_item = MLX5_FLOW_LAYER_GRE;
1405                         break;
1406                 case RTE_FLOW_ITEM_TYPE_MPLS:
1407                         ret = mlx5_flow_validate_item_mpls(dev, items,
1408                                                            item_flags,
1409                                                            last_item, error);
1410                         if (ret < 0)
1411                                 return ret;
1412                         last_item = MLX5_FLOW_LAYER_MPLS;
1413                         break;
1414                 case RTE_FLOW_ITEM_TYPE_ICMP:
1415                 case RTE_FLOW_ITEM_TYPE_ICMP6:
1416                         return rte_flow_error_set(error, ENOTSUP,
1417                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1418                                                   NULL, "ICMP/ICMP6 "
1419                                                   "item not supported");
1420                 default:
1421                         return rte_flow_error_set(error, ENOTSUP,
1422                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1423                                                   NULL, "item not supported");
1424                 }
1425                 item_flags |= last_item;
1426         }
1427         if (is_empty_vlan)
1428                 return rte_flow_error_set(error, ENOTSUP,
1429                                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1430                     "VLAN matching without vid specification is not supported");
1431         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1432                 switch (actions->type) {
1433                 case RTE_FLOW_ACTION_TYPE_VOID:
1434                         break;
1435                 case RTE_FLOW_ACTION_TYPE_FLAG:
1436                         ret = mlx5_flow_validate_action_flag(action_flags,
1437                                                              attr,
1438                                                              error);
1439                         if (ret < 0)
1440                                 return ret;
1441                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1442                         break;
1443                 case RTE_FLOW_ACTION_TYPE_MARK:
1444                         ret = mlx5_flow_validate_action_mark(actions,
1445                                                              action_flags,
1446                                                              attr,
1447                                                              error);
1448                         if (ret < 0)
1449                                 return ret;
1450                         action_flags |= MLX5_FLOW_ACTION_MARK;
1451                         break;
1452                 case RTE_FLOW_ACTION_TYPE_DROP:
1453                         ret = mlx5_flow_validate_action_drop(action_flags,
1454                                                              attr,
1455                                                              error);
1456                         if (ret < 0)
1457                                 return ret;
1458                         action_flags |= MLX5_FLOW_ACTION_DROP;
1459                         break;
1460                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1461                         ret = mlx5_flow_validate_action_queue(actions,
1462                                                               action_flags, dev,
1463                                                               attr,
1464                                                               error);
1465                         if (ret < 0)
1466                                 return ret;
1467                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1468                         break;
1469                 case RTE_FLOW_ACTION_TYPE_RSS:
1470                         ret = mlx5_flow_validate_action_rss(actions,
1471                                                             action_flags, dev,
1472                                                             attr, item_flags,
1473                                                             error);
1474                         if (ret < 0)
1475                                 return ret;
1476                         action_flags |= MLX5_FLOW_ACTION_RSS;
1477                         break;
1478                 case RTE_FLOW_ACTION_TYPE_COUNT:
1479                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1480                         if (ret < 0)
1481                                 return ret;
1482                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1483                         break;
1484                 default:
1485                         return rte_flow_error_set(error, ENOTSUP,
1486                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1487                                                   actions,
1488                                                   "action not supported");
1489                 }
1490         }
1491         /*
1492          * Validate the drop action mutual exclusion with other actions.
1493          * Drop action is mutually-exclusive with any other action, except for
1494          * Count action.
1495          */
1496         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1497             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1498                 return rte_flow_error_set(error, EINVAL,
1499                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1500                                           "Drop action is mutually-exclusive "
1501                                           "with any other action, except for "
1502                                           "Count action");
1503         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1504                 return rte_flow_error_set(error, EINVAL,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1506                                           "no fate action is found");
1507         return 0;
1508 }
1509
1510 /**
1511  * Calculate the required bytes that are needed for the action part of the verbs
1512  * flow.
1513  *
1514  * @param[in] actions
1515  *   Pointer to the list of actions.
1516  *
1517  * @return
1518  *   The size of the memory needed for all actions.
1519  */
1520 static int
1521 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1522 {
1523         int size = 0;
1524
1525         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1526                 switch (actions->type) {
1527                 case RTE_FLOW_ACTION_TYPE_VOID:
1528                         break;
1529                 case RTE_FLOW_ACTION_TYPE_FLAG:
1530                         size += sizeof(struct ibv_flow_spec_action_tag);
1531                         break;
1532                 case RTE_FLOW_ACTION_TYPE_MARK:
1533                         size += sizeof(struct ibv_flow_spec_action_tag);
1534                         break;
1535                 case RTE_FLOW_ACTION_TYPE_DROP:
1536                         size += sizeof(struct ibv_flow_spec_action_drop);
1537                         break;
1538                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1539                         break;
1540                 case RTE_FLOW_ACTION_TYPE_RSS:
1541                         break;
1542                 case RTE_FLOW_ACTION_TYPE_COUNT:
1543 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1544         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1545                         size += sizeof(struct ibv_flow_spec_counter_action);
1546 #endif
1547                         break;
1548                 default:
1549                         break;
1550                 }
1551         }
1552         return size;
1553 }
1554
1555 /**
1556  * Calculate the required bytes that are needed for the item part of the verbs
1557  * flow.
1558  *
1559  * @param[in] items
1560  *   Pointer to the list of items.
1561  *
1562  * @return
1563  *   The size of the memory needed for all items.
1564  */
1565 static int
1566 flow_verbs_get_items_size(const struct rte_flow_item items[])
1567 {
1568         int size = 0;
1569
1570         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1571                 switch (items->type) {
1572                 case RTE_FLOW_ITEM_TYPE_VOID:
1573                         break;
1574                 case RTE_FLOW_ITEM_TYPE_ETH:
1575                         size += sizeof(struct ibv_flow_spec_eth);
1576                         break;
1577                 case RTE_FLOW_ITEM_TYPE_VLAN:
1578                         size += sizeof(struct ibv_flow_spec_eth);
1579                         break;
1580                 case RTE_FLOW_ITEM_TYPE_IPV4:
1581                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1582                         break;
1583                 case RTE_FLOW_ITEM_TYPE_IPV6:
1584                         size += sizeof(struct ibv_flow_spec_ipv6);
1585                         break;
1586                 case RTE_FLOW_ITEM_TYPE_UDP:
1587                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1588                         break;
1589                 case RTE_FLOW_ITEM_TYPE_TCP:
1590                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1591                         break;
1592                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1593                         size += sizeof(struct ibv_flow_spec_tunnel);
1594                         break;
1595                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1596                         size += sizeof(struct ibv_flow_spec_tunnel);
1597                         break;
1598 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1599                 case RTE_FLOW_ITEM_TYPE_GRE:
1600                         size += sizeof(struct ibv_flow_spec_gre);
1601                         break;
1602                 case RTE_FLOW_ITEM_TYPE_MPLS:
1603                         size += sizeof(struct ibv_flow_spec_mpls);
1604                         break;
1605 #else
1606                 case RTE_FLOW_ITEM_TYPE_GRE:
1607                         size += sizeof(struct ibv_flow_spec_tunnel);
1608                         break;
1609 #endif
1610                 default:
1611                         break;
1612                 }
1613         }
1614         return size;
1615 }
1616
1617 /**
1618  * Internal preparation function. Allocate mlx5_flow with the required size.
1619  * The required size is calculate based on the actions and items. This function
1620  * also returns the detected actions and items for later use.
1621  *
1622  * @param[in] dev
1623  *   Pointer to Ethernet device.
1624  * @param[in] attr
1625  *   Pointer to the flow attributes.
1626  * @param[in] items
1627  *   Pointer to the list of items.
1628  * @param[in] actions
1629  *   Pointer to the list of actions.
1630  * @param[out] error
1631  *   Pointer to the error structure.
1632  *
1633  * @return
1634  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1635  *   is set.
1636  */
1637 static struct mlx5_flow *
1638 flow_verbs_prepare(struct rte_eth_dev *dev,
1639                    const struct rte_flow_attr *attr __rte_unused,
1640                    const struct rte_flow_item items[],
1641                    const struct rte_flow_action actions[],
1642                    struct rte_flow_error *error)
1643 {
1644         size_t size = 0;
1645         uint32_t handle_idx = 0;
1646         struct mlx5_flow *dev_flow;
1647         struct mlx5_flow_handle *dev_handle;
1648         struct mlx5_priv *priv = dev->data->dev_private;
1649         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1650
1651         MLX5_ASSERT(wks);
1652         size += flow_verbs_get_actions_size(actions);
1653         size += flow_verbs_get_items_size(items);
1654         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1655                 rte_flow_error_set(error, E2BIG,
1656                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1657                                    "Verbs spec/action size too large");
1658                 return NULL;
1659         }
1660         /* In case of corrupting the memory. */
1661         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1662                 rte_flow_error_set(error, ENOSPC,
1663                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1664                                    "not free temporary device flow");
1665                 return NULL;
1666         }
1667         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1668                                    &handle_idx);
1669         if (!dev_handle) {
1670                 rte_flow_error_set(error, ENOMEM,
1671                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1672                                    "not enough memory to create flow handle");
1673                 return NULL;
1674         }
1675         MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
1676         dev_flow = &wks->flows[wks->flow_idx++];
1677         dev_flow->handle = dev_handle;
1678         dev_flow->handle_idx = handle_idx;
1679         /* Memcpy is used, only size needs to be cleared to 0. */
1680         dev_flow->verbs.size = 0;
1681         dev_flow->verbs.attr.num_of_specs = 0;
1682         dev_flow->ingress = attr->ingress;
1683         dev_flow->hash_fields = 0;
1684         /* Need to set transfer attribute: not supported in Verbs mode. */
1685         return dev_flow;
1686 }
1687
1688 /**
1689  * Fill the flow with verb spec.
1690  *
1691  * @param[in] dev
1692  *   Pointer to Ethernet device.
1693  * @param[in, out] dev_flow
1694  *   Pointer to the mlx5 flow.
1695  * @param[in] attr
1696  *   Pointer to the flow attributes.
1697  * @param[in] items
1698  *   Pointer to the list of items.
1699  * @param[in] actions
1700  *   Pointer to the list of actions.
1701  * @param[out] error
1702  *   Pointer to the error structure.
1703  *
1704  * @return
1705  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1706  */
1707 static int
1708 flow_verbs_translate(struct rte_eth_dev *dev,
1709                      struct mlx5_flow *dev_flow,
1710                      const struct rte_flow_attr *attr,
1711                      const struct rte_flow_item items[],
1712                      const struct rte_flow_action actions[],
1713                      struct rte_flow_error *error)
1714 {
1715         uint64_t item_flags = 0;
1716         uint64_t action_flags = 0;
1717         uint64_t priority = attr->priority;
1718         uint32_t subpriority = 0;
1719         struct mlx5_priv *priv = dev->data->dev_private;
1720         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1721         struct mlx5_flow_rss_desc *rss_desc;
1722
1723         MLX5_ASSERT(wks);
1724         rss_desc = &wks->rss_desc;
1725         if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1726                 priority = priv->config.flow_prio - 1;
1727         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1728                 int ret;
1729
1730                 switch (actions->type) {
1731                 case RTE_FLOW_ACTION_TYPE_VOID:
1732                         break;
1733                 case RTE_FLOW_ACTION_TYPE_FLAG:
1734                         flow_verbs_translate_action_flag(dev_flow, actions);
1735                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1736                         dev_flow->handle->mark = 1;
1737                         break;
1738                 case RTE_FLOW_ACTION_TYPE_MARK:
1739                         flow_verbs_translate_action_mark(dev_flow, actions);
1740                         action_flags |= MLX5_FLOW_ACTION_MARK;
1741                         dev_flow->handle->mark = 1;
1742                         break;
1743                 case RTE_FLOW_ACTION_TYPE_DROP:
1744                         flow_verbs_translate_action_drop(dev_flow, actions);
1745                         action_flags |= MLX5_FLOW_ACTION_DROP;
1746                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1747                         break;
1748                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1749                         flow_verbs_translate_action_queue(rss_desc, actions);
1750                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1751                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1752                         break;
1753                 case RTE_FLOW_ACTION_TYPE_RSS:
1754                         flow_verbs_translate_action_rss(rss_desc, actions);
1755                         action_flags |= MLX5_FLOW_ACTION_RSS;
1756                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1757                         break;
1758                 case RTE_FLOW_ACTION_TYPE_COUNT:
1759                         ret = flow_verbs_translate_action_count(dev_flow,
1760                                                                 actions,
1761                                                                 dev, error);
1762                         if (ret < 0)
1763                                 return ret;
1764                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1765                         break;
1766                 default:
1767                         return rte_flow_error_set(error, ENOTSUP,
1768                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1769                                                   actions,
1770                                                   "action not supported");
1771                 }
1772         }
1773         dev_flow->act_flags = action_flags;
1774         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1775                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1776
1777                 switch (items->type) {
1778                 case RTE_FLOW_ITEM_TYPE_VOID:
1779                         break;
1780                 case RTE_FLOW_ITEM_TYPE_ETH:
1781                         flow_verbs_translate_item_eth(dev_flow, items,
1782                                                       item_flags);
1783                         subpriority = MLX5_PRIORITY_MAP_L2;
1784                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1785                                                MLX5_FLOW_LAYER_OUTER_L2;
1786                         break;
1787                 case RTE_FLOW_ITEM_TYPE_VLAN:
1788                         flow_verbs_translate_item_vlan(dev_flow, items,
1789                                                        item_flags);
1790                         subpriority = MLX5_PRIORITY_MAP_L2;
1791                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1792                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1793                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1794                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1795                         break;
1796                 case RTE_FLOW_ITEM_TYPE_IPV4:
1797                         flow_verbs_translate_item_ipv4(dev_flow, items,
1798                                                        item_flags);
1799                         subpriority = MLX5_PRIORITY_MAP_L3;
1800                         dev_flow->hash_fields |=
1801                                 mlx5_flow_hashfields_adjust
1802                                         (rss_desc, tunnel,
1803                                          MLX5_IPV4_LAYER_TYPES,
1804                                          MLX5_IPV4_IBV_RX_HASH);
1805                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1806                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1807                         break;
1808                 case RTE_FLOW_ITEM_TYPE_IPV6:
1809                         flow_verbs_translate_item_ipv6(dev_flow, items,
1810                                                        item_flags);
1811                         subpriority = MLX5_PRIORITY_MAP_L3;
1812                         dev_flow->hash_fields |=
1813                                 mlx5_flow_hashfields_adjust
1814                                         (rss_desc, tunnel,
1815                                          MLX5_IPV6_LAYER_TYPES,
1816                                          MLX5_IPV6_IBV_RX_HASH);
1817                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1818                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1819                         break;
1820                 case RTE_FLOW_ITEM_TYPE_TCP:
1821                         flow_verbs_translate_item_tcp(dev_flow, items,
1822                                                       item_flags);
1823                         subpriority = MLX5_PRIORITY_MAP_L4;
1824                         if (dev_flow->hash_fields != 0)
1825                                 dev_flow->hash_fields |=
1826                                         mlx5_flow_hashfields_adjust
1827                                         (rss_desc, tunnel, RTE_ETH_RSS_TCP,
1828                                          (IBV_RX_HASH_SRC_PORT_TCP |
1829                                           IBV_RX_HASH_DST_PORT_TCP));
1830                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1831                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1832                         break;
1833                 case RTE_FLOW_ITEM_TYPE_UDP:
1834                         flow_verbs_translate_item_udp(dev_flow, items,
1835                                                       item_flags);
1836                         subpriority = MLX5_PRIORITY_MAP_L4;
1837                         if (dev_flow->hash_fields != 0)
1838                                 dev_flow->hash_fields |=
1839                                         mlx5_flow_hashfields_adjust
1840                                         (rss_desc, tunnel, RTE_ETH_RSS_UDP,
1841                                          (IBV_RX_HASH_SRC_PORT_UDP |
1842                                           IBV_RX_HASH_DST_PORT_UDP));
1843                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1844                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1845                         break;
1846                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1847                         flow_verbs_translate_item_vxlan(dev_flow, items,
1848                                                         item_flags);
1849                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1850                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1851                         break;
1852                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1853                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1854                                                             item_flags);
1855                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1856                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1857                         break;
1858                 case RTE_FLOW_ITEM_TYPE_GRE:
1859                         flow_verbs_translate_item_gre(dev_flow, items,
1860                                                       item_flags);
1861                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1862                         item_flags |= MLX5_FLOW_LAYER_GRE;
1863                         break;
1864                 case RTE_FLOW_ITEM_TYPE_MPLS:
1865                         flow_verbs_translate_item_mpls(dev_flow, items,
1866                                                        item_flags);
1867                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1868                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1869                         break;
1870                 default:
1871                         return rte_flow_error_set(error, ENOTSUP,
1872                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1873                                                   NULL, "item not supported");
1874                 }
1875         }
1876         dev_flow->handle->layers = item_flags;
1877         /* Other members of attr will be ignored. */
1878         dev_flow->verbs.attr.priority =
1879                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1880         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1881         return 0;
1882 }
1883
1884 /**
1885  * Remove the flow from the NIC but keeps it in memory.
1886  *
1887  * @param[in] dev
1888  *   Pointer to the Ethernet device structure.
1889  * @param[in, out] flow
1890  *   Pointer to flow structure.
1891  */
1892 static void
1893 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1894 {
1895         struct mlx5_priv *priv = dev->data->dev_private;
1896         struct mlx5_flow_handle *handle;
1897         uint32_t handle_idx;
1898
1899         if (!flow)
1900                 return;
1901         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1902                        handle_idx, handle, next) {
1903                 if (handle->drv_flow) {
1904                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1905                         handle->drv_flow = NULL;
1906                 }
1907                 /* hrxq is union, don't touch it only the flag is set. */
1908                 if (handle->rix_hrxq &&
1909                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1910                         mlx5_hrxq_release(dev, handle->rix_hrxq);
1911                         handle->rix_hrxq = 0;
1912                 }
1913                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1914                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1915         }
1916 }
1917
1918 /**
1919  * Remove the flow from the NIC and the memory.
1920  *
1921  * @param[in] dev
1922  *   Pointer to the Ethernet device structure.
1923  * @param[in, out] flow
1924  *   Pointer to flow structure.
1925  */
1926 static void
1927 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1928 {
1929         struct mlx5_priv *priv = dev->data->dev_private;
1930         struct mlx5_flow_handle *handle;
1931
1932         if (!flow)
1933                 return;
1934         flow_verbs_remove(dev, flow);
1935         while (flow->dev_handles) {
1936                 uint32_t tmp_idx = flow->dev_handles;
1937
1938                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1939                                    tmp_idx);
1940                 if (!handle)
1941                         return;
1942                 flow->dev_handles = handle->next.next;
1943                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1944                            tmp_idx);
1945         }
1946         if (flow->counter) {
1947                 flow_verbs_counter_release(dev, flow->counter);
1948                 flow->counter = 0;
1949         }
1950 }
1951
1952 /**
1953  * Apply the flow to the NIC.
1954  *
1955  * @param[in] dev
1956  *   Pointer to the Ethernet device structure.
1957  * @param[in, out] flow
1958  *   Pointer to flow structure.
1959  * @param[out] error
1960  *   Pointer to error structure.
1961  *
1962  * @return
1963  *   0 on success, a negative errno value otherwise and rte_errno is set.
1964  */
1965 static int
1966 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1967                  struct rte_flow_error *error)
1968 {
1969         struct mlx5_priv *priv = dev->data->dev_private;
1970         struct mlx5_flow_handle *handle;
1971         struct mlx5_flow *dev_flow;
1972         struct mlx5_hrxq *hrxq;
1973         uint32_t dev_handles;
1974         int err;
1975         int idx;
1976         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1977
1978         MLX5_ASSERT(wks);
1979         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
1980                 dev_flow = &wks->flows[idx];
1981                 handle = dev_flow->handle;
1982                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1983                         MLX5_ASSERT(priv->drop_queue.hrxq);
1984                         hrxq = priv->drop_queue.hrxq;
1985                 } else {
1986                         uint32_t hrxq_idx;
1987                         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
1988
1989                         MLX5_ASSERT(rss_desc->queue_num);
1990                         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
1991                         rss_desc->hash_fields = dev_flow->hash_fields;
1992                         rss_desc->tunnel = !!(handle->layers &
1993                                               MLX5_FLOW_LAYER_TUNNEL);
1994                         rss_desc->shared_rss = 0;
1995                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
1996                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1997                                               hrxq_idx);
1998                         if (!hrxq) {
1999                                 rte_flow_error_set
2000                                         (error, rte_errno,
2001                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2002                                          "cannot get hash queue");
2003                                 goto error;
2004                         }
2005                         handle->rix_hrxq = hrxq_idx;
2006                 }
2007                 MLX5_ASSERT(hrxq);
2008                 handle->drv_flow = mlx5_glue->create_flow
2009                                         (hrxq->qp, &dev_flow->verbs.attr);
2010                 if (!handle->drv_flow) {
2011                         rte_flow_error_set(error, errno,
2012                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2013                                            NULL,
2014                                            "hardware refuses to create flow");
2015                         goto error;
2016                 }
2017                 if (priv->vmwa_context &&
2018                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2019                         /*
2020                          * The rule contains the VLAN pattern.
2021                          * For VF we are going to create VLAN
2022                          * interface to make hypervisor set correct
2023                          * e-Switch vport context.
2024                          */
2025                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2026                 }
2027         }
2028         return 0;
2029 error:
2030         err = rte_errno; /* Save rte_errno before cleanup. */
2031         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2032                        dev_handles, handle, next) {
2033                 /* hrxq is union, don't touch it only the flag is set. */
2034                 if (handle->rix_hrxq &&
2035                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
2036                         mlx5_hrxq_release(dev, handle->rix_hrxq);
2037                         handle->rix_hrxq = 0;
2038                 }
2039                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2040                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2041         }
2042         rte_errno = err; /* Restore rte_errno. */
2043         return -rte_errno;
2044 }
2045
2046 /**
2047  * Query a flow.
2048  *
2049  * @see rte_flow_query()
2050  * @see rte_flow_ops
2051  */
2052 static int
2053 flow_verbs_query(struct rte_eth_dev *dev,
2054                  struct rte_flow *flow,
2055                  const struct rte_flow_action *actions,
2056                  void *data,
2057                  struct rte_flow_error *error)
2058 {
2059         int ret = -EINVAL;
2060
2061         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2062                 switch (actions->type) {
2063                 case RTE_FLOW_ACTION_TYPE_VOID:
2064                         break;
2065                 case RTE_FLOW_ACTION_TYPE_COUNT:
2066                         ret = flow_verbs_counter_query(dev, flow, data, error);
2067                         break;
2068                 default:
2069                         return rte_flow_error_set(error, ENOTSUP,
2070                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2071                                                   actions,
2072                                                   "action not supported");
2073                 }
2074         }
2075         return ret;
2076 }
2077
2078 static int
2079 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains,
2080                        uint32_t flags)
2081 {
2082         RTE_SET_USED(dev);
2083         RTE_SET_USED(domains);
2084         RTE_SET_USED(flags);
2085
2086         return 0;
2087 }
2088
2089 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2090         .validate = flow_verbs_validate,
2091         .prepare = flow_verbs_prepare,
2092         .translate = flow_verbs_translate,
2093         .apply = flow_verbs_apply,
2094         .remove = flow_verbs_remove,
2095         .destroy = flow_verbs_destroy,
2096         .query = flow_verbs_query,
2097         .sync_domain = flow_verbs_sync_domain,
2098 };