drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rxtx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /**
  43  * Discover the maximum number of priority available.
  44  *
  45  * @param[in] dev
  46  *   Pointer to the Ethernet device structure.
  47  *
  48  * @return
  49  *   number of supported flow priority on success, a negative errno
  50  *   value otherwise and rte_errno is set.
  51  */
  52 int
  53 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  54 {
  55         struct mlx5_priv *priv = dev->data->dev_private;
  56         struct {
  57                 struct ibv_flow_attr attr;
  58                 struct ibv_flow_spec_eth eth;
  59                 struct ibv_flow_spec_action_drop drop;
  60         } flow_attr = {
  61                 .attr = {
  62                         .num_of_specs = 2,
  63                         .port = (uint8_t)priv->dev_port,
  64                 },
  65                 .eth = {
  66                         .type = IBV_FLOW_SPEC_ETH,
  67                         .size = sizeof(struct ibv_flow_spec_eth),
  68                 },
  69                 .drop = {
  70                         .size = sizeof(struct ibv_flow_spec_action_drop),
  71                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  72                 },
  73         };
  74         struct ibv_flow *flow;
  75         struct mlx5_hrxq *drop = mlx5_drop_action_create(dev);
  76         uint16_t vprio[] = { 8, 16 };
  77         int i;
  78         int priority = 0;
  79
  80         if (!drop) {
  81                 rte_errno = ENOTSUP;
  82                 return -rte_errno;
  83         }
  84         for (i = 0; i != RTE_DIM(vprio); i++) {
  85                 flow_attr.attr.priority = vprio[i] - 1;
  86                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  87                 if (!flow)
  88                         break;
  89                 claim_zero(mlx5_glue->destroy_flow(flow));
  90                 priority = vprio[i];
  91         }
  92         mlx5_drop_action_destroy(dev);
  93         switch (priority) {
  94         case 8:
  95                 priority = RTE_DIM(priority_map_3);
  96                 break;
  97         case 16:
  98                 priority = RTE_DIM(priority_map_5);
  99                 break;
 100         default:
 101                 rte_errno = ENOTSUP;
 102                 DRV_LOG(ERR,
 103                         "port %u verbs maximum priority: %d expected 8/16",
 104                         dev->data->port_id, priority);
 105                 return -rte_errno;
 106         }
 107         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 108                 dev->data->port_id, priority);
 109         return priority;
 110 }
 111
 112 /**
 113  * Adjust flow priority based on the highest layer and the request priority.
 114  *
 115  * @param[in] dev
 116  *   Pointer to the Ethernet device structure.
 117  * @param[in] priority
 118  *   The rule base priority.
 119  * @param[in] subpriority
 120  *   The priority based on the items.
 121  *
 122  * @return
 123  *   The new priority.
 124  */
 125 uint32_t
 126 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 127                                    uint32_t subpriority)
 128 {
 129         uint32_t res = 0;
 130         struct mlx5_priv *priv = dev->data->dev_private;
 131
 132         switch (priv->config.flow_prio) {
 133         case RTE_DIM(priority_map_3):
 134                 res = priority_map_3[priority][subpriority];
 135                 break;
 136         case RTE_DIM(priority_map_5):
 137                 res = priority_map_5[priority][subpriority];
 138                 break;
 139         }
 140         return  res;
 141 }
 142
 143 /**
 144  * Get Verbs flow counter by index.
 145  *
 146  * @param[in] dev
 147  *   Pointer to the Ethernet device structure.
 148  * @param[in] idx
 149  *   mlx5 flow counter index in the container.
 150  * @param[out] ppool
 151  *   mlx5 flow counter pool in the container,
 152  *
 153  * @return
 154  *   A pointer to the counter, NULL otherwise.
 155  */
 156 static struct mlx5_flow_counter *
 157 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 158                               uint32_t idx,
 159                               struct mlx5_flow_counter_pool **ppool)
 160 {
 161         struct mlx5_priv *priv = dev->data->dev_private;
 162         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
 163         struct mlx5_flow_counter_pool *pool;
 164
 165         idx--;
 166         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
 167         MLX5_ASSERT(pool);
 168         if (ppool)
 169                 *ppool = pool;
 170         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 171 }
 172
 173 /**
 174  * Create Verbs flow counter with Verbs library.
 175  *
 176  * @param[in] dev
 177  *   Pointer to the Ethernet device structure.
 178  * @param[in, out] counter
 179  *   mlx5 flow counter object, contains the counter id,
 180  *   handle of created Verbs flow counter is returned
 181  *   in cs field (if counters are supported).
 182  *
 183  * @return
 184  *   0 On success else a negative errno value is returned
 185  *   and rte_errno is set.
 186  */
 187 static int
 188 flow_verbs_counter_create(struct rte_eth_dev *dev,
 189                           struct mlx5_flow_counter_ext *counter)
 190 {
 191 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 192         struct mlx5_priv *priv = dev->data->dev_private;
 193         struct ibv_context *ctx = priv->sh->ctx;
 194         struct ibv_counter_set_init_attr init = {
 195                          .counter_set_id = counter->id};
 196
 197         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
 198         if (!counter->cs) {
 199                 rte_errno = ENOTSUP;
 200                 return -ENOTSUP;
 201         }
 202         return 0;
 203 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 204         struct mlx5_priv *priv = dev->data->dev_private;
 205         struct ibv_context *ctx = priv->sh->ctx;
 206         struct ibv_counters_init_attr init = {0};
 207         struct ibv_counter_attach_attr attach;
 208         int ret;
 209
 210         memset(&attach, 0, sizeof(attach));
 211         counter->cs = mlx5_glue->create_counters(ctx, &init);
 212         if (!counter->cs) {
 213                 rte_errno = ENOTSUP;
 214                 return -ENOTSUP;
 215         }
 216         attach.counter_desc = IBV_COUNTER_PACKETS;
 217         attach.index = 0;
 218         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
 219         if (!ret) {
 220                 attach.counter_desc = IBV_COUNTER_BYTES;
 221                 attach.index = 1;
 222                 ret = mlx5_glue->attach_counters
 223                                         (counter->cs, &attach, NULL);
 224         }
 225         if (ret) {
 226                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 227                 counter->cs = NULL;
 228                 rte_errno = ret;
 229                 return -ret;
 230         }
 231         return 0;
 232 #else
 233         (void)dev;
 234         (void)counter;
 235         rte_errno = ENOTSUP;
 236         return -ENOTSUP;
 237 #endif
 238 }
 239
 240 /**
 241  * Get a flow counter.
 242  *
 243  * @param[in] dev
 244  *   Pointer to the Ethernet device structure.
 245  * @param[in] shared
 246  *   Indicate if this counter is shared with other flows.
 247  * @param[in] id
 248  *   Counter identifier.
 249  *
 250  * @return
 251  *   Index to the counter, 0 otherwise and rte_errno is set.
 252  */
 253 static uint32_t
 254 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 255 {
 256         struct mlx5_priv *priv = dev->data->dev_private;
 257         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
 258         struct mlx5_flow_counter_pool *pool = NULL;
 259         struct mlx5_flow_counter_ext *cnt_ext = NULL;
 260         struct mlx5_flow_counter *cnt = NULL;
 261         uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
 262         uint32_t pool_idx;
 263         uint32_t i;
 264         int ret;
 265
 266         if (shared) {
 267                 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 268                         pool = cont->pools[pool_idx];
 269                         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 270                                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
 271                                 if (cnt_ext->shared && cnt_ext->id == id) {
 272                                         cnt_ext->ref_cnt++;
 273                                         return MLX5_MAKE_CNT_IDX(pool_idx, i);
 274                                 }
 275                         }
 276                 }
 277         }
 278         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 279                 pool = cont->pools[pool_idx];
 280                 if (!pool)
 281                         continue;
 282                 cnt = TAILQ_FIRST(&pool->counters[0]);
 283                 if (cnt)
 284                         break;
 285         }
 286         if (!cnt) {
 287                 struct mlx5_flow_counter_pool **pools;
 288                 uint32_t size;
 289
 290                 if (n_valid == cont->n) {
 291                         /* Resize the container pool array. */
 292                         size = sizeof(struct mlx5_flow_counter_pool *) *
 293                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 294                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 295                                             SOCKET_ID_ANY);
 296                         if (!pools)
 297                                 return 0;
 298                         if (n_valid) {
 299                                 memcpy(pools, cont->pools,
 300                                        sizeof(struct mlx5_flow_counter_pool *) *
 301                                        n_valid);
 302                                 mlx5_free(cont->pools);
 303                         }
 304                         cont->pools = pools;
 305                         cont->n += MLX5_CNT_CONTAINER_RESIZE;
 306                 }
 307                 /* Allocate memory for new pool*/
 308                 size = sizeof(*pool) + (sizeof(*cnt_ext) + sizeof(*cnt)) *
 309                        MLX5_COUNTERS_PER_POOL;
 310                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 311                 if (!pool)
 312                         return 0;
 313                 pool->type |= CNT_POOL_TYPE_EXT;
 314                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 315                         cnt = MLX5_POOL_GET_CNT(pool, i);
 316                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 317                 }
 318                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 319                 cont->pools[n_valid] = pool;
 320                 pool_idx = n_valid;
 321                 rte_atomic16_add(&cont->n_valid, 1);
 322                 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
 323         }
 324         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 325         cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
 326         cnt_ext->id = id;
 327         cnt_ext->shared = shared;
 328         cnt_ext->ref_cnt = 1;
 329         cnt->hits = 0;
 330         cnt->bytes = 0;
 331         /* Create counter with Verbs. */
 332         ret = flow_verbs_counter_create(dev, cnt_ext);
 333         if (!ret) {
 334                 TAILQ_REMOVE(&pool->counters[0], cnt, next);
 335                 return MLX5_MAKE_CNT_IDX(pool_idx, i);
 336         }
 337         /* Some error occurred in Verbs library. */
 338         rte_errno = -ret;
 339         return 0;
 340 }
 341
 342 /**
 343  * Release a flow counter.
 344  *
 345  * @param[in] dev
 346  *   Pointer to the Ethernet device structure.
 347  * @param[in] counter
 348  *   Index to the counter handler.
 349  */
 350 static void
 351 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 352 {
 353         struct mlx5_flow_counter_pool *pool;
 354         struct mlx5_flow_counter *cnt;
 355         struct mlx5_flow_counter_ext *cnt_ext;
 356
 357         cnt = flow_verbs_counter_get_by_idx(dev, counter,
 358                                             &pool);
 359         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
 360         if (--cnt_ext->ref_cnt == 0) {
 361 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 362                 claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
 363                 cnt_ext->cs = NULL;
 364 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 365                 claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs));
 366                 cnt_ext->cs = NULL;
 367 #endif
 368                 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 369         }
 370 }
 371
 372 /**
 373  * Query a flow counter via Verbs library call.
 374  *
 375  * @see rte_flow_query()
 376  * @see rte_flow_ops
 377  */
 378 static int
 379 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 380                          struct rte_flow *flow, void *data,
 381                          struct rte_flow_error *error)
 382 {
 383 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 384         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 385         if (flow->counter) {
 386                 struct mlx5_flow_counter_pool *pool;
 387                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 388                                                 (dev, flow->counter, &pool);
 389                 struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
 390                                                 (pool, cnt);
 391                 struct rte_flow_query_count *qc = data;
 392                 uint64_t counters[2] = {0, 0};
 393 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 394                 struct ibv_query_counter_set_attr query_cs_attr = {
 395                         .cs = cnt_ext->cs,
 396                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 397                 };
 398                 struct ibv_counter_set_data query_out = {
 399                         .out = counters,
 400                         .outlen = 2 * sizeof(uint64_t),
 401                 };
 402                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 403                                                        &query_out);
 404 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 405                 int err = mlx5_glue->query_counters
 406                                (cnt_ext->cs, counters,
 407                                 RTE_DIM(counters),
 408                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 409 #endif
 410                 if (err)
 411                         return rte_flow_error_set
 412                                 (error, err,
 413                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 414                                  NULL,
 415                                  "cannot read counter");
 416                 qc->hits_set = 1;
 417                 qc->bytes_set = 1;
 418                 qc->hits = counters[0] - cnt->hits;
 419                 qc->bytes = counters[1] - cnt->bytes;
 420                 if (qc->reset) {
 421                         cnt->hits = counters[0];
 422                         cnt->bytes = counters[1];
 423                 }
 424                 return 0;
 425         }
 426         return rte_flow_error_set(error, EINVAL,
 427                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 428                                   NULL,
 429                                   "flow does not have counter");
 430 #else
 431         (void)flow;
 432         (void)data;
 433         return rte_flow_error_set(error, ENOTSUP,
 434                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 435                                   NULL,
 436                                   "counters are not available");
 437 #endif
 438 }
 439
 440 /**
 441  * Add a verbs item specification into @p verbs.
 442  *
 443  * @param[out] verbs
 444  *   Pointer to verbs structure.
 445  * @param[in] src
 446  *   Create specification.
 447  * @param[in] size
 448  *   Size in bytes of the specification to copy.
 449  */
 450 static void
 451 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 452                     void *src, unsigned int size)
 453 {
 454         void *dst;
 455
 456         if (!verbs)
 457                 return;
 458         MLX5_ASSERT(verbs->specs);
 459         dst = (void *)(verbs->specs + verbs->size);
 460         memcpy(dst, src, size);
 461         ++verbs->attr.num_of_specs;
 462         verbs->size += size;
 463 }
 464
 465 /**
 466  * Convert the @p item into a Verbs specification. This function assumes that
 467  * the input is valid and that there is space to insert the requested item
 468  * into the flow.
 469  *
 470  * @param[in, out] dev_flow
 471  *   Pointer to dev_flow structure.
 472  * @param[in] item
 473  *   Item specification.
 474  * @param[in] item_flags
 475  *   Parsed item flags.
 476  */
 477 static void
 478 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 479                               const struct rte_flow_item *item,
 480                               uint64_t item_flags)
 481 {
 482         const struct rte_flow_item_eth *spec = item->spec;
 483         const struct rte_flow_item_eth *mask = item->mask;
 484         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 485         struct ibv_flow_spec_eth eth = {
 486                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 487                 .size = size,
 488         };
 489
 490         if (!mask)
 491                 mask = &rte_flow_item_eth_mask;
 492         if (spec) {
 493                 unsigned int i;
 494
 495                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 496                         RTE_ETHER_ADDR_LEN);
 497                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 498                         RTE_ETHER_ADDR_LEN);
 499                 eth.val.ether_type = spec->type;
 500                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 501                         RTE_ETHER_ADDR_LEN);
 502                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 503                         RTE_ETHER_ADDR_LEN);
 504                 eth.mask.ether_type = mask->type;
 505                 /* Remove unwanted bits from values. */
 506                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 507                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 508                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 509                 }
 510                 eth.val.ether_type &= eth.mask.ether_type;
 511         }
 512         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 513 }
 514
 515 /**
 516  * Update the VLAN tag in the Verbs Ethernet specification.
 517  * This function assumes that the input is valid and there is space to add
 518  * the requested item.
 519  *
 520  * @param[in, out] attr
 521  *   Pointer to Verbs attributes structure.
 522  * @param[in] eth
 523  *   Verbs structure containing the VLAN information to copy.
 524  */
 525 static void
 526 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 527                             struct ibv_flow_spec_eth *eth)
 528 {
 529         unsigned int i;
 530         const enum ibv_flow_spec_type search = eth->type;
 531         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 532                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 533
 534         for (i = 0; i != attr->num_of_specs; ++i) {
 535                 if (hdr->type == search) {
 536                         struct ibv_flow_spec_eth *e =
 537                                 (struct ibv_flow_spec_eth *)hdr;
 538
 539                         e->val.vlan_tag = eth->val.vlan_tag;
 540                         e->mask.vlan_tag = eth->mask.vlan_tag;
 541                         e->val.ether_type = eth->val.ether_type;
 542                         e->mask.ether_type = eth->mask.ether_type;
 543                         break;
 544                 }
 545                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 546         }
 547 }
 548
 549 /**
 550  * Convert the @p item into a Verbs specification. This function assumes that
 551  * the input is valid and that there is space to insert the requested item
 552  * into the flow.
 553  *
 554  * @param[in, out] dev_flow
 555  *   Pointer to dev_flow structure.
 556  * @param[in] item
 557  *   Item specification.
 558  * @param[in] item_flags
 559  *   Parsed item flags.
 560  */
 561 static void
 562 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 563                                const struct rte_flow_item *item,
 564                                uint64_t item_flags)
 565 {
 566         const struct rte_flow_item_vlan *spec = item->spec;
 567         const struct rte_flow_item_vlan *mask = item->mask;
 568         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 569         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 570         struct ibv_flow_spec_eth eth = {
 571                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 572                 .size = size,
 573         };
 574         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 575                                       MLX5_FLOW_LAYER_OUTER_L2;
 576
 577         if (!mask)
 578                 mask = &rte_flow_item_vlan_mask;
 579         if (spec) {
 580                 eth.val.vlan_tag = spec->tci;
 581                 eth.mask.vlan_tag = mask->tci;
 582                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 583                 eth.val.ether_type = spec->inner_type;
 584                 eth.mask.ether_type = mask->inner_type;
 585                 eth.val.ether_type &= eth.mask.ether_type;
 586         }
 587         if (!(item_flags & l2m))
 588                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 589         else
 590                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 591         if (!tunnel)
 592                 dev_flow->handle->vf_vlan.tag =
 593                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 594 }
 595
 596 /**
 597  * Convert the @p item into a Verbs specification. This function assumes that
 598  * the input is valid and that there is space to insert the requested item
 599  * into the flow.
 600  *
 601  * @param[in, out] dev_flow
 602  *   Pointer to dev_flow structure.
 603  * @param[in] item
 604  *   Item specification.
 605  * @param[in] item_flags
 606  *   Parsed item flags.
 607  */
 608 static void
 609 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 610                                const struct rte_flow_item *item,
 611                                uint64_t item_flags)
 612 {
 613         const struct rte_flow_item_ipv4 *spec = item->spec;
 614         const struct rte_flow_item_ipv4 *mask = item->mask;
 615         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 616         struct ibv_flow_spec_ipv4_ext ipv4 = {
 617                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 618                 .size = size,
 619         };
 620
 621         if (!mask)
 622                 mask = &rte_flow_item_ipv4_mask;
 623         if (spec) {
 624                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 625                         .src_ip = spec->hdr.src_addr,
 626                         .dst_ip = spec->hdr.dst_addr,
 627                         .proto = spec->hdr.next_proto_id,
 628                         .tos = spec->hdr.type_of_service,
 629                 };
 630                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 631                         .src_ip = mask->hdr.src_addr,
 632                         .dst_ip = mask->hdr.dst_addr,
 633                         .proto = mask->hdr.next_proto_id,
 634                         .tos = mask->hdr.type_of_service,
 635                 };
 636                 /* Remove unwanted bits from values. */
 637                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 638                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 639                 ipv4.val.proto &= ipv4.mask.proto;
 640                 ipv4.val.tos &= ipv4.mask.tos;
 641         }
 642         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 643 }
 644
 645 /**
 646  * Convert the @p item into a Verbs specification. This function assumes that
 647  * the input is valid and that there is space to insert the requested item
 648  * into the flow.
 649  *
 650  * @param[in, out] dev_flow
 651  *   Pointer to dev_flow structure.
 652  * @param[in] item
 653  *   Item specification.
 654  * @param[in] item_flags
 655  *   Parsed item flags.
 656  */
 657 static void
 658 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 659                                const struct rte_flow_item *item,
 660                                uint64_t item_flags)
 661 {
 662         const struct rte_flow_item_ipv6 *spec = item->spec;
 663         const struct rte_flow_item_ipv6 *mask = item->mask;
 664         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 665         struct ibv_flow_spec_ipv6 ipv6 = {
 666                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 667                 .size = size,
 668         };
 669
 670         if (!mask)
 671                 mask = &rte_flow_item_ipv6_mask;
 672         if (spec) {
 673                 unsigned int i;
 674                 uint32_t vtc_flow_val;
 675                 uint32_t vtc_flow_mask;
 676
 677                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 678                        RTE_DIM(ipv6.val.src_ip));
 679                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 680                        RTE_DIM(ipv6.val.dst_ip));
 681                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 682                        RTE_DIM(ipv6.mask.src_ip));
 683                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 684                        RTE_DIM(ipv6.mask.dst_ip));
 685                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 686                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 687                 ipv6.val.flow_label =
 688                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 689                                          RTE_IPV6_HDR_FL_SHIFT);
 690                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 691                                          RTE_IPV6_HDR_TC_SHIFT;
 692                 ipv6.val.next_hdr = spec->hdr.proto;
 693                 ipv6.mask.flow_label =
 694                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 695                                          RTE_IPV6_HDR_FL_SHIFT);
 696                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 697                                           RTE_IPV6_HDR_TC_SHIFT;
 698                 ipv6.mask.next_hdr = mask->hdr.proto;
 699                 /* Remove unwanted bits from values. */
 700                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 701                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 702                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 703                 }
 704                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 705                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 706                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 707         }
 708         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 709 }
 710
 711 /**
 712  * Convert the @p item into a Verbs specification. This function assumes that
 713  * the input is valid and that there is space to insert the requested item
 714  * into the flow.
 715  *
 716  * @param[in, out] dev_flow
 717  *   Pointer to dev_flow structure.
 718  * @param[in] item
 719  *   Item specification.
 720  * @param[in] item_flags
 721  *   Parsed item flags.
 722  */
 723 static void
 724 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 725                               const struct rte_flow_item *item,
 726                               uint64_t item_flags __rte_unused)
 727 {
 728         const struct rte_flow_item_tcp *spec = item->spec;
 729         const struct rte_flow_item_tcp *mask = item->mask;
 730         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 731         struct ibv_flow_spec_tcp_udp tcp = {
 732                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 733                 .size = size,
 734         };
 735
 736         if (!mask)
 737                 mask = &rte_flow_item_tcp_mask;
 738         if (spec) {
 739                 tcp.val.dst_port = spec->hdr.dst_port;
 740                 tcp.val.src_port = spec->hdr.src_port;
 741                 tcp.mask.dst_port = mask->hdr.dst_port;
 742                 tcp.mask.src_port = mask->hdr.src_port;
 743                 /* Remove unwanted bits from values. */
 744                 tcp.val.src_port &= tcp.mask.src_port;
 745                 tcp.val.dst_port &= tcp.mask.dst_port;
 746         }
 747         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 748 }
 749
 750 /**
 751  * Convert the @p item into a Verbs specification. This function assumes that
 752  * the input is valid and that there is space to insert the requested item
 753  * into the flow.
 754  *
 755  * @param[in, out] dev_flow
 756  *   Pointer to dev_flow structure.
 757  * @param[in] item
 758  *   Item specification.
 759  * @param[in] item_flags
 760  *   Parsed item flags.
 761  */
 762 static void
 763 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 764                               const struct rte_flow_item *item,
 765                               uint64_t item_flags __rte_unused)
 766 {
 767         const struct rte_flow_item_udp *spec = item->spec;
 768         const struct rte_flow_item_udp *mask = item->mask;
 769         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 770         struct ibv_flow_spec_tcp_udp udp = {
 771                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 772                 .size = size,
 773         };
 774
 775         if (!mask)
 776                 mask = &rte_flow_item_udp_mask;
 777         if (spec) {
 778                 udp.val.dst_port = spec->hdr.dst_port;
 779                 udp.val.src_port = spec->hdr.src_port;
 780                 udp.mask.dst_port = mask->hdr.dst_port;
 781                 udp.mask.src_port = mask->hdr.src_port;
 782                 /* Remove unwanted bits from values. */
 783                 udp.val.src_port &= udp.mask.src_port;
 784                 udp.val.dst_port &= udp.mask.dst_port;
 785         }
 786         item++;
 787         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 788                 item++;
 789         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 790                 switch ((item)->type) {
 791                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 792                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 793                         udp.mask.dst_port = 0xffff;
 794                         break;
 795                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 796                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 797                         udp.mask.dst_port = 0xffff;
 798                         break;
 799                 case RTE_FLOW_ITEM_TYPE_MPLS:
 800                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 801                         udp.mask.dst_port = 0xffff;
 802                         break;
 803                 default:
 804                         break;
 805                 }
 806         }
 807
 808         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 809 }
 810
 811 /**
 812  * Convert the @p item into a Verbs specification. This function assumes that
 813  * the input is valid and that there is space to insert the requested item
 814  * into the flow.
 815  *
 816  * @param[in, out] dev_flow
 817  *   Pointer to dev_flow structure.
 818  * @param[in] item
 819  *   Item specification.
 820  * @param[in] item_flags
 821  *   Parsed item flags.
 822  */
 823 static void
 824 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 825                                 const struct rte_flow_item *item,
 826                                 uint64_t item_flags __rte_unused)
 827 {
 828         const struct rte_flow_item_vxlan *spec = item->spec;
 829         const struct rte_flow_item_vxlan *mask = item->mask;
 830         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 831         struct ibv_flow_spec_tunnel vxlan = {
 832                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 833                 .size = size,
 834         };
 835         union vni {
 836                 uint32_t vlan_id;
 837                 uint8_t vni[4];
 838         } id = { .vlan_id = 0, };
 839
 840         if (!mask)
 841                 mask = &rte_flow_item_vxlan_mask;
 842         if (spec) {
 843                 memcpy(&id.vni[1], spec->vni, 3);
 844                 vxlan.val.tunnel_id = id.vlan_id;
 845                 memcpy(&id.vni[1], mask->vni, 3);
 846                 vxlan.mask.tunnel_id = id.vlan_id;
 847                 /* Remove unwanted bits from values. */
 848                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 849         }
 850         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 851 }
 852
 853 /**
 854  * Convert the @p item into a Verbs specification. This function assumes that
 855  * the input is valid and that there is space to insert the requested item
 856  * into the flow.
 857  *
 858  * @param[in, out] dev_flow
 859  *   Pointer to dev_flow structure.
 860  * @param[in] item
 861  *   Item specification.
 862  * @param[in] item_flags
 863  *   Parsed item flags.
 864  */
 865 static void
 866 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 867                                     const struct rte_flow_item *item,
 868                                     uint64_t item_flags __rte_unused)
 869 {
 870         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 871         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 872         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 873         struct ibv_flow_spec_tunnel vxlan_gpe = {
 874                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 875                 .size = size,
 876         };
 877         union vni {
 878                 uint32_t vlan_id;
 879                 uint8_t vni[4];
 880         } id = { .vlan_id = 0, };
 881
 882         if (!mask)
 883                 mask = &rte_flow_item_vxlan_gpe_mask;
 884         if (spec) {
 885                 memcpy(&id.vni[1], spec->vni, 3);
 886                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 887                 memcpy(&id.vni[1], mask->vni, 3);
 888                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 889                 /* Remove unwanted bits from values. */
 890                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 891         }
 892         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 893 }
 894
 895 /**
 896  * Update the protocol in Verbs IPv4/IPv6 spec.
 897  *
 898  * @param[in, out] attr
 899  *   Pointer to Verbs attributes structure.
 900  * @param[in] search
 901  *   Specification type to search in order to update the IP protocol.
 902  * @param[in] protocol
 903  *   Protocol value to set if none is present in the specification.
 904  */
 905 static void
 906 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 907                                        enum ibv_flow_spec_type search,
 908                                        uint8_t protocol)
 909 {
 910         unsigned int i;
 911         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 912                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 913
 914         if (!attr)
 915                 return;
 916         for (i = 0; i != attr->num_of_specs; ++i) {
 917                 if (hdr->type == search) {
 918                         union {
 919                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 920                                 struct ibv_flow_spec_ipv6 *ipv6;
 921                         } ip;
 922
 923                         switch (search) {
 924                         case IBV_FLOW_SPEC_IPV4_EXT:
 925                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 926                                 if (!ip.ipv4->val.proto) {
 927                                         ip.ipv4->val.proto = protocol;
 928                                         ip.ipv4->mask.proto = 0xff;
 929                                 }
 930                                 break;
 931                         case IBV_FLOW_SPEC_IPV6:
 932                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 933                                 if (!ip.ipv6->val.next_hdr) {
 934                                         ip.ipv6->val.next_hdr = protocol;
 935                                         ip.ipv6->mask.next_hdr = 0xff;
 936                                 }
 937                                 break;
 938                         default:
 939                                 break;
 940                         }
 941                         break;
 942                 }
 943                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 944         }
 945 }
 946
 947 /**
 948  * Convert the @p item into a Verbs specification. This function assumes that
 949  * the input is valid and that there is space to insert the requested item
 950  * into the flow.
 951  *
 952  * @param[in, out] dev_flow
 953  *   Pointer to dev_flow structure.
 954  * @param[in] item
 955  *   Item specification.
 956  * @param[in] item_flags
 957  *   Parsed item flags.
 958  */
 959 static void
 960 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 961                               const struct rte_flow_item *item __rte_unused,
 962                               uint64_t item_flags)
 963 {
 964         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 965 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 966         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 967         struct ibv_flow_spec_tunnel tunnel = {
 968                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 969                 .size = size,
 970         };
 971 #else
 972         const struct rte_flow_item_gre *spec = item->spec;
 973         const struct rte_flow_item_gre *mask = item->mask;
 974         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 975         struct ibv_flow_spec_gre tunnel = {
 976                 .type = IBV_FLOW_SPEC_GRE,
 977                 .size = size,
 978         };
 979
 980         if (!mask)
 981                 mask = &rte_flow_item_gre_mask;
 982         if (spec) {
 983                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 984                 tunnel.val.protocol = spec->protocol;
 985                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 986                 tunnel.mask.protocol = mask->protocol;
 987                 /* Remove unwanted bits from values. */
 988                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 989                 tunnel.val.protocol &= tunnel.mask.protocol;
 990                 tunnel.val.key &= tunnel.mask.key;
 991         }
 992 #endif
 993         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 994                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 995                                                        IBV_FLOW_SPEC_IPV4_EXT,
 996                                                        IPPROTO_GRE);
 997         else
 998                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 999                                                        IBV_FLOW_SPEC_IPV6,
1000                                                        IPPROTO_GRE);
1001         flow_verbs_spec_add(verbs, &tunnel, size);
1002 }
1003
1004 /**
1005  * Convert the @p action into a Verbs specification. This function assumes that
1006  * the input is valid and that there is space to insert the requested action
1007  * into the flow. This function also return the action that was added.
1008  *
1009  * @param[in, out] dev_flow
1010  *   Pointer to dev_flow structure.
1011  * @param[in] item
1012  *   Item specification.
1013  * @param[in] item_flags
1014  *   Parsed item flags.
1015  */
1016 static void
1017 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1018                                const struct rte_flow_item *item __rte_unused,
1019                                uint64_t item_flags __rte_unused)
1020 {
1021 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1022         const struct rte_flow_item_mpls *spec = item->spec;
1023         const struct rte_flow_item_mpls *mask = item->mask;
1024         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1025         struct ibv_flow_spec_mpls mpls = {
1026                 .type = IBV_FLOW_SPEC_MPLS,
1027                 .size = size,
1028         };
1029
1030         if (!mask)
1031                 mask = &rte_flow_item_mpls_mask;
1032         if (spec) {
1033                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1034                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1035                 /* Remove unwanted bits from values.  */
1036                 mpls.val.label &= mpls.mask.label;
1037         }
1038         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1039 #endif
1040 }
1041
1042 /**
1043  * Convert the @p action into a Verbs specification. This function assumes that
1044  * the input is valid and that there is space to insert the requested action
1045  * into the flow.
1046  *
1047  * @param[in] dev_flow
1048  *   Pointer to mlx5_flow.
1049  * @param[in] action
1050  *   Action configuration.
1051  */
1052 static void
1053 flow_verbs_translate_action_drop
1054         (struct mlx5_flow *dev_flow,
1055          const struct rte_flow_action *action __rte_unused)
1056 {
1057         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1058         struct ibv_flow_spec_action_drop drop = {
1059                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1060                         .size = size,
1061         };
1062
1063         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1064 }
1065
1066 /**
1067  * Convert the @p action into a Verbs specification. This function assumes that
1068  * the input is valid and that there is space to insert the requested action
1069  * into the flow.
1070  *
1071  * @param[in] rss_desc
1072  *   Pointer to mlx5_flow_rss_desc.
1073  * @param[in] action
1074  *   Action configuration.
1075  */
1076 static void
1077 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1078                                   const struct rte_flow_action *action)
1079 {
1080         const struct rte_flow_action_queue *queue = action->conf;
1081
1082         rss_desc->queue[0] = queue->index;
1083         rss_desc->queue_num = 1;
1084 }
1085
1086 /**
1087  * Convert the @p action into a Verbs specification. This function assumes that
1088  * the input is valid and that there is space to insert the requested action
1089  * into the flow.
1090  *
1091  * @param[in] rss_desc
1092  *   Pointer to mlx5_flow_rss_desc.
1093  * @param[in] action
1094  *   Action configuration.
1095  */
1096 static void
1097 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1098                                 const struct rte_flow_action *action)
1099 {
1100         const struct rte_flow_action_rss *rss = action->conf;
1101         const uint8_t *rss_key;
1102
1103         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1104         rss_desc->queue_num = rss->queue_num;
1105         /* NULL RSS key indicates default RSS key. */
1106         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1107         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1108         /*
1109          * rss->level and rss.types should be set in advance when expanding
1110          * items for RSS.
1111          */
1112 }
1113
1114 /**
1115  * Convert the @p action into a Verbs specification. This function assumes that
1116  * the input is valid and that there is space to insert the requested action
1117  * into the flow.
1118  *
1119  * @param[in] dev_flow
1120  *   Pointer to mlx5_flow.
1121  * @param[in] action
1122  *   Action configuration.
1123  */
1124 static void
1125 flow_verbs_translate_action_flag
1126         (struct mlx5_flow *dev_flow,
1127          const struct rte_flow_action *action __rte_unused)
1128 {
1129         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1130         struct ibv_flow_spec_action_tag tag = {
1131                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1132                 .size = size,
1133                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1134         };
1135
1136         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1137 }
1138
1139 /**
1140  * Convert the @p action into a Verbs specification. This function assumes that
1141  * the input is valid and that there is space to insert the requested action
1142  * into the flow.
1143  *
1144  * @param[in] dev_flow
1145  *   Pointer to mlx5_flow.
1146  * @param[in] action
1147  *   Action configuration.
1148  */
1149 static void
1150 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1151                                  const struct rte_flow_action *action)
1152 {
1153         const struct rte_flow_action_mark *mark = action->conf;
1154         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1155         struct ibv_flow_spec_action_tag tag = {
1156                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1157                 .size = size,
1158                 .tag_id = mlx5_flow_mark_set(mark->id),
1159         };
1160
1161         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1162 }
1163
1164 /**
1165  * Convert the @p action into a Verbs specification. This function assumes that
1166  * the input is valid and that there is space to insert the requested action
1167  * into the flow.
1168  *
1169  * @param[in] dev
1170  *   Pointer to the Ethernet device structure.
1171  * @param[in] action
1172  *   Action configuration.
1173  * @param[in] dev_flow
1174  *   Pointer to mlx5_flow.
1175  * @param[out] error
1176  *   Pointer to error structure.
1177  *
1178  * @return
1179  *   0 On success else a negative errno value is returned and rte_errno is set.
1180  */
1181 static int
1182 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1183                                   const struct rte_flow_action *action,
1184                                   struct rte_eth_dev *dev,
1185                                   struct rte_flow_error *error)
1186 {
1187         const struct rte_flow_action_count *count = action->conf;
1188         struct rte_flow *flow = dev_flow->flow;
1189 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1190         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1191         struct mlx5_flow_counter_pool *pool;
1192         struct mlx5_flow_counter *cnt = NULL;
1193         struct mlx5_flow_counter_ext *cnt_ext;
1194         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1195         struct ibv_flow_spec_counter_action counter = {
1196                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1197                 .size = size,
1198         };
1199 #endif
1200
1201         if (!flow->counter) {
1202                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1203                                                        count->id);
1204                 if (!flow->counter)
1205                         return rte_flow_error_set(error, rte_errno,
1206                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1207                                                   action,
1208                                                   "cannot get counter"
1209                                                   " context.");
1210         }
1211 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1212         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1213         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1214         counter.counter_set_handle = cnt_ext->cs->handle;
1215         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1216 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1217         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1218         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1219         counter.counters = cnt_ext->cs;
1220         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1221 #endif
1222         return 0;
1223 }
1224
1225 /**
1226  * Internal validation function. For validating both actions and items.
1227  *
1228  * @param[in] dev
1229  *   Pointer to the Ethernet device structure.
1230  * @param[in] attr
1231  *   Pointer to the flow attributes.
1232  * @param[in] items
1233  *   Pointer to the list of items.
1234  * @param[in] actions
1235  *   Pointer to the list of actions.
1236  * @param[in] external
1237  *   This flow rule is created by request external to PMD.
1238  * @param[in] hairpin
1239  *   Number of hairpin TX actions, 0 means classic flow.
1240  * @param[out] error
1241  *   Pointer to the error structure.
1242  *
1243  * @return
1244  *   0 on success, a negative errno value otherwise and rte_errno is set.
1245  */
1246 static int
1247 flow_verbs_validate(struct rte_eth_dev *dev,
1248                     const struct rte_flow_attr *attr,
1249                     const struct rte_flow_item items[],
1250                     const struct rte_flow_action actions[],
1251                     bool external __rte_unused,
1252                     int hairpin __rte_unused,
1253                     struct rte_flow_error *error)
1254 {
1255         int ret;
1256         uint64_t action_flags = 0;
1257         uint64_t item_flags = 0;
1258         uint64_t last_item = 0;
1259         uint8_t next_protocol = 0xff;
1260         uint16_t ether_type = 0;
1261
1262         if (items == NULL)
1263                 return -1;
1264         ret = mlx5_flow_validate_attributes(dev, attr, error);
1265         if (ret < 0)
1266                 return ret;
1267         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1268                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1269                 int ret = 0;
1270
1271                 switch (items->type) {
1272                 case RTE_FLOW_ITEM_TYPE_VOID:
1273                         break;
1274                 case RTE_FLOW_ITEM_TYPE_ETH:
1275                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1276                                                           error);
1277                         if (ret < 0)
1278                                 return ret;
1279                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1280                                              MLX5_FLOW_LAYER_OUTER_L2;
1281                         if (items->mask != NULL && items->spec != NULL) {
1282                                 ether_type =
1283                                         ((const struct rte_flow_item_eth *)
1284                                          items->spec)->type;
1285                                 ether_type &=
1286                                         ((const struct rte_flow_item_eth *)
1287                                          items->mask)->type;
1288                                 ether_type = rte_be_to_cpu_16(ether_type);
1289                         } else {
1290                                 ether_type = 0;
1291                         }
1292                         break;
1293                 case RTE_FLOW_ITEM_TYPE_VLAN:
1294                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1295                                                            dev, error);
1296                         if (ret < 0)
1297                                 return ret;
1298                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1299                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1300                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1301                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1302                         if (items->mask != NULL && items->spec != NULL) {
1303                                 ether_type =
1304                                         ((const struct rte_flow_item_vlan *)
1305                                          items->spec)->inner_type;
1306                                 ether_type &=
1307                                         ((const struct rte_flow_item_vlan *)
1308                                          items->mask)->inner_type;
1309                                 ether_type = rte_be_to_cpu_16(ether_type);
1310                         } else {
1311                                 ether_type = 0;
1312                         }
1313                         break;
1314                 case RTE_FLOW_ITEM_TYPE_IPV4:
1315                         ret = mlx5_flow_validate_item_ipv4
1316                                                 (items, item_flags,
1317                                                  last_item, ether_type, NULL,
1318                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1319                                                  error);
1320                         if (ret < 0)
1321                                 return ret;
1322                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1323                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1324                         if (items->mask != NULL &&
1325                             ((const struct rte_flow_item_ipv4 *)
1326                              items->mask)->hdr.next_proto_id) {
1327                                 next_protocol =
1328                                         ((const struct rte_flow_item_ipv4 *)
1329                                          (items->spec))->hdr.next_proto_id;
1330                                 next_protocol &=
1331                                         ((const struct rte_flow_item_ipv4 *)
1332                                          (items->mask))->hdr.next_proto_id;
1333                         } else {
1334                                 /* Reset for inner layer. */
1335                                 next_protocol = 0xff;
1336                         }
1337                         break;
1338                 case RTE_FLOW_ITEM_TYPE_IPV6:
1339                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1340                                                            last_item,
1341                                                            ether_type, NULL,
1342                                                            error);
1343                         if (ret < 0)
1344                                 return ret;
1345                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1346                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1347                         if (items->mask != NULL &&
1348                             ((const struct rte_flow_item_ipv6 *)
1349                              items->mask)->hdr.proto) {
1350                                 next_protocol =
1351                                         ((const struct rte_flow_item_ipv6 *)
1352                                          items->spec)->hdr.proto;
1353                                 next_protocol &=
1354                                         ((const struct rte_flow_item_ipv6 *)
1355                                          items->mask)->hdr.proto;
1356                         } else {
1357                                 /* Reset for inner layer. */
1358                                 next_protocol = 0xff;
1359                         }
1360                         break;
1361                 case RTE_FLOW_ITEM_TYPE_UDP:
1362                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1363                                                           next_protocol,
1364                                                           error);
1365                         if (ret < 0)
1366                                 return ret;
1367                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1368                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1369                         break;
1370                 case RTE_FLOW_ITEM_TYPE_TCP:
1371                         ret = mlx5_flow_validate_item_tcp
1372                                                 (items, item_flags,
1373                                                  next_protocol,
1374                                                  &rte_flow_item_tcp_mask,
1375                                                  error);
1376                         if (ret < 0)
1377                                 return ret;
1378                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1379                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1380                         break;
1381                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1382                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1383                                                             error);
1384                         if (ret < 0)
1385                                 return ret;
1386                         last_item = MLX5_FLOW_LAYER_VXLAN;
1387                         break;
1388                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1389                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1390                                                                 item_flags,
1391                                                                 dev, error);
1392                         if (ret < 0)
1393                                 return ret;
1394                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1395                         break;
1396                 case RTE_FLOW_ITEM_TYPE_GRE:
1397                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1398                                                           next_protocol, error);
1399                         if (ret < 0)
1400                                 return ret;
1401                         last_item = MLX5_FLOW_LAYER_GRE;
1402                         break;
1403                 case RTE_FLOW_ITEM_TYPE_MPLS:
1404                         ret = mlx5_flow_validate_item_mpls(dev, items,
1405                                                            item_flags,
1406                                                            last_item, error);
1407                         if (ret < 0)
1408                                 return ret;
1409                         last_item = MLX5_FLOW_LAYER_MPLS;
1410                         break;
1411                 default:
1412                         return rte_flow_error_set(error, ENOTSUP,
1413                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1414                                                   NULL, "item not supported");
1415                 }
1416                 item_flags |= last_item;
1417         }
1418         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1419                 switch (actions->type) {
1420                 case RTE_FLOW_ACTION_TYPE_VOID:
1421                         break;
1422                 case RTE_FLOW_ACTION_TYPE_FLAG:
1423                         ret = mlx5_flow_validate_action_flag(action_flags,
1424                                                              attr,
1425                                                              error);
1426                         if (ret < 0)
1427                                 return ret;
1428                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1429                         break;
1430                 case RTE_FLOW_ACTION_TYPE_MARK:
1431                         ret = mlx5_flow_validate_action_mark(actions,
1432                                                              action_flags,
1433                                                              attr,
1434                                                              error);
1435                         if (ret < 0)
1436                                 return ret;
1437                         action_flags |= MLX5_FLOW_ACTION_MARK;
1438                         break;
1439                 case RTE_FLOW_ACTION_TYPE_DROP:
1440                         ret = mlx5_flow_validate_action_drop(action_flags,
1441                                                              attr,
1442                                                              error);
1443                         if (ret < 0)
1444                                 return ret;
1445                         action_flags |= MLX5_FLOW_ACTION_DROP;
1446                         break;
1447                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1448                         ret = mlx5_flow_validate_action_queue(actions,
1449                                                               action_flags, dev,
1450                                                               attr,
1451                                                               error);
1452                         if (ret < 0)
1453                                 return ret;
1454                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1455                         break;
1456                 case RTE_FLOW_ACTION_TYPE_RSS:
1457                         ret = mlx5_flow_validate_action_rss(actions,
1458                                                             action_flags, dev,
1459                                                             attr, item_flags,
1460                                                             error);
1461                         if (ret < 0)
1462                                 return ret;
1463                         action_flags |= MLX5_FLOW_ACTION_RSS;
1464                         break;
1465                 case RTE_FLOW_ACTION_TYPE_COUNT:
1466                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1467                         if (ret < 0)
1468                                 return ret;
1469                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1470                         break;
1471                 default:
1472                         return rte_flow_error_set(error, ENOTSUP,
1473                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1474                                                   actions,
1475                                                   "action not supported");
1476                 }
1477         }
1478         /*
1479          * Validate the drop action mutual exclusion with other actions.
1480          * Drop action is mutually-exclusive with any other action, except for
1481          * Count action.
1482          */
1483         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1484             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1485                 return rte_flow_error_set(error, EINVAL,
1486                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1487                                           "Drop action is mutually-exclusive "
1488                                           "with any other action, except for "
1489                                           "Count action");
1490         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1491                 return rte_flow_error_set(error, EINVAL,
1492                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1493                                           "no fate action is found");
1494         return 0;
1495 }
1496
1497 /**
1498  * Calculate the required bytes that are needed for the action part of the verbs
1499  * flow.
1500  *
1501  * @param[in] actions
1502  *   Pointer to the list of actions.
1503  *
1504  * @return
1505  *   The size of the memory needed for all actions.
1506  */
1507 static int
1508 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1509 {
1510         int size = 0;
1511
1512         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1513                 switch (actions->type) {
1514                 case RTE_FLOW_ACTION_TYPE_VOID:
1515                         break;
1516                 case RTE_FLOW_ACTION_TYPE_FLAG:
1517                         size += sizeof(struct ibv_flow_spec_action_tag);
1518                         break;
1519                 case RTE_FLOW_ACTION_TYPE_MARK:
1520                         size += sizeof(struct ibv_flow_spec_action_tag);
1521                         break;
1522                 case RTE_FLOW_ACTION_TYPE_DROP:
1523                         size += sizeof(struct ibv_flow_spec_action_drop);
1524                         break;
1525                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1526                         break;
1527                 case RTE_FLOW_ACTION_TYPE_RSS:
1528                         break;
1529                 case RTE_FLOW_ACTION_TYPE_COUNT:
1530 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1531         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1532                         size += sizeof(struct ibv_flow_spec_counter_action);
1533 #endif
1534                         break;
1535                 default:
1536                         break;
1537                 }
1538         }
1539         return size;
1540 }
1541
1542 /**
1543  * Calculate the required bytes that are needed for the item part of the verbs
1544  * flow.
1545  *
1546  * @param[in] items
1547  *   Pointer to the list of items.
1548  *
1549  * @return
1550  *   The size of the memory needed for all items.
1551  */
1552 static int
1553 flow_verbs_get_items_size(const struct rte_flow_item items[])
1554 {
1555         int size = 0;
1556
1557         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1558                 switch (items->type) {
1559                 case RTE_FLOW_ITEM_TYPE_VOID:
1560                         break;
1561                 case RTE_FLOW_ITEM_TYPE_ETH:
1562                         size += sizeof(struct ibv_flow_spec_eth);
1563                         break;
1564                 case RTE_FLOW_ITEM_TYPE_VLAN:
1565                         size += sizeof(struct ibv_flow_spec_eth);
1566                         break;
1567                 case RTE_FLOW_ITEM_TYPE_IPV4:
1568                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1569                         break;
1570                 case RTE_FLOW_ITEM_TYPE_IPV6:
1571                         size += sizeof(struct ibv_flow_spec_ipv6);
1572                         break;
1573                 case RTE_FLOW_ITEM_TYPE_UDP:
1574                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1575                         break;
1576                 case RTE_FLOW_ITEM_TYPE_TCP:
1577                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1578                         break;
1579                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1580                         size += sizeof(struct ibv_flow_spec_tunnel);
1581                         break;
1582                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1583                         size += sizeof(struct ibv_flow_spec_tunnel);
1584                         break;
1585 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1586                 case RTE_FLOW_ITEM_TYPE_GRE:
1587                         size += sizeof(struct ibv_flow_spec_gre);
1588                         break;
1589                 case RTE_FLOW_ITEM_TYPE_MPLS:
1590                         size += sizeof(struct ibv_flow_spec_mpls);
1591                         break;
1592 #else
1593                 case RTE_FLOW_ITEM_TYPE_GRE:
1594                         size += sizeof(struct ibv_flow_spec_tunnel);
1595                         break;
1596 #endif
1597                 default:
1598                         break;
1599                 }
1600         }
1601         return size;
1602 }
1603
1604 /**
1605  * Internal preparation function. Allocate mlx5_flow with the required size.
1606  * The required size is calculate based on the actions and items. This function
1607  * also returns the detected actions and items for later use.
1608  *
1609  * @param[in] dev
1610  *   Pointer to Ethernet device.
1611  * @param[in] attr
1612  *   Pointer to the flow attributes.
1613  * @param[in] items
1614  *   Pointer to the list of items.
1615  * @param[in] actions
1616  *   Pointer to the list of actions.
1617  * @param[out] error
1618  *   Pointer to the error structure.
1619  *
1620  * @return
1621  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1622  *   is set.
1623  */
1624 static struct mlx5_flow *
1625 flow_verbs_prepare(struct rte_eth_dev *dev,
1626                    const struct rte_flow_attr *attr __rte_unused,
1627                    const struct rte_flow_item items[],
1628                    const struct rte_flow_action actions[],
1629                    struct rte_flow_error *error)
1630 {
1631         size_t size = 0;
1632         uint32_t handle_idx = 0;
1633         struct mlx5_flow *dev_flow;
1634         struct mlx5_flow_handle *dev_handle;
1635         struct mlx5_priv *priv = dev->data->dev_private;
1636
1637         size += flow_verbs_get_actions_size(actions);
1638         size += flow_verbs_get_items_size(items);
1639         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1640                 rte_flow_error_set(error, E2BIG,
1641                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1642                                    "Verbs spec/action size too large");
1643                 return NULL;
1644         }
1645         /* In case of corrupting the memory. */
1646         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1647                 rte_flow_error_set(error, ENOSPC,
1648                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1649                                    "not free temporary device flow");
1650                 return NULL;
1651         }
1652         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1653                                    &handle_idx);
1654         if (!dev_handle) {
1655                 rte_flow_error_set(error, ENOMEM,
1656                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1657                                    "not enough memory to create flow handle");
1658                 return NULL;
1659         }
1660         /* No multi-thread supporting. */
1661         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1662         dev_flow->handle = dev_handle;
1663         dev_flow->handle_idx = handle_idx;
1664         /* Memcpy is used, only size needs to be cleared to 0. */
1665         dev_flow->verbs.size = 0;
1666         dev_flow->verbs.attr.num_of_specs = 0;
1667         dev_flow->ingress = attr->ingress;
1668         dev_flow->hash_fields = 0;
1669         /* Need to set transfer attribute: not supported in Verbs mode. */
1670         return dev_flow;
1671 }
1672
1673 /**
1674  * Fill the flow with verb spec.
1675  *
1676  * @param[in] dev
1677  *   Pointer to Ethernet device.
1678  * @param[in, out] dev_flow
1679  *   Pointer to the mlx5 flow.
1680  * @param[in] attr
1681  *   Pointer to the flow attributes.
1682  * @param[in] items
1683  *   Pointer to the list of items.
1684  * @param[in] actions
1685  *   Pointer to the list of actions.
1686  * @param[out] error
1687  *   Pointer to the error structure.
1688  *
1689  * @return
1690  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1691  */
1692 static int
1693 flow_verbs_translate(struct rte_eth_dev *dev,
1694                      struct mlx5_flow *dev_flow,
1695                      const struct rte_flow_attr *attr,
1696                      const struct rte_flow_item items[],
1697                      const struct rte_flow_action actions[],
1698                      struct rte_flow_error *error)
1699 {
1700         uint64_t item_flags = 0;
1701         uint64_t action_flags = 0;
1702         uint64_t priority = attr->priority;
1703         uint32_t subpriority = 0;
1704         struct mlx5_priv *priv = dev->data->dev_private;
1705         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
1706                                               priv->rss_desc)
1707                                               [!!priv->flow_nested_idx];
1708
1709         if (priority == MLX5_FLOW_PRIO_RSVD)
1710                 priority = priv->config.flow_prio - 1;
1711         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1712                 int ret;
1713
1714                 switch (actions->type) {
1715                 case RTE_FLOW_ACTION_TYPE_VOID:
1716                         break;
1717                 case RTE_FLOW_ACTION_TYPE_FLAG:
1718                         flow_verbs_translate_action_flag(dev_flow, actions);
1719                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1720                         dev_flow->handle->mark = 1;
1721                         break;
1722                 case RTE_FLOW_ACTION_TYPE_MARK:
1723                         flow_verbs_translate_action_mark(dev_flow, actions);
1724                         action_flags |= MLX5_FLOW_ACTION_MARK;
1725                         dev_flow->handle->mark = 1;
1726                         break;
1727                 case RTE_FLOW_ACTION_TYPE_DROP:
1728                         flow_verbs_translate_action_drop(dev_flow, actions);
1729                         action_flags |= MLX5_FLOW_ACTION_DROP;
1730                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1731                         break;
1732                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1733                         flow_verbs_translate_action_queue(rss_desc, actions);
1734                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1735                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1736                         break;
1737                 case RTE_FLOW_ACTION_TYPE_RSS:
1738                         flow_verbs_translate_action_rss(rss_desc, actions);
1739                         action_flags |= MLX5_FLOW_ACTION_RSS;
1740                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1741                         break;
1742                 case RTE_FLOW_ACTION_TYPE_COUNT:
1743                         ret = flow_verbs_translate_action_count(dev_flow,
1744                                                                 actions,
1745                                                                 dev, error);
1746                         if (ret < 0)
1747                                 return ret;
1748                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1749                         break;
1750                 default:
1751                         return rte_flow_error_set(error, ENOTSUP,
1752                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1753                                                   actions,
1754                                                   "action not supported");
1755                 }
1756         }
1757         dev_flow->act_flags = action_flags;
1758         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1759                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1760
1761                 switch (items->type) {
1762                 case RTE_FLOW_ITEM_TYPE_VOID:
1763                         break;
1764                 case RTE_FLOW_ITEM_TYPE_ETH:
1765                         flow_verbs_translate_item_eth(dev_flow, items,
1766                                                       item_flags);
1767                         subpriority = MLX5_PRIORITY_MAP_L2;
1768                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1769                                                MLX5_FLOW_LAYER_OUTER_L2;
1770                         break;
1771                 case RTE_FLOW_ITEM_TYPE_VLAN:
1772                         flow_verbs_translate_item_vlan(dev_flow, items,
1773                                                        item_flags);
1774                         subpriority = MLX5_PRIORITY_MAP_L2;
1775                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1776                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1777                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1778                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1779                         break;
1780                 case RTE_FLOW_ITEM_TYPE_IPV4:
1781                         flow_verbs_translate_item_ipv4(dev_flow, items,
1782                                                        item_flags);
1783                         subpriority = MLX5_PRIORITY_MAP_L3;
1784                         dev_flow->hash_fields |=
1785                                 mlx5_flow_hashfields_adjust
1786                                         (rss_desc, tunnel,
1787                                          MLX5_IPV4_LAYER_TYPES,
1788                                          MLX5_IPV4_IBV_RX_HASH);
1789                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1790                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1791                         break;
1792                 case RTE_FLOW_ITEM_TYPE_IPV6:
1793                         flow_verbs_translate_item_ipv6(dev_flow, items,
1794                                                        item_flags);
1795                         subpriority = MLX5_PRIORITY_MAP_L3;
1796                         dev_flow->hash_fields |=
1797                                 mlx5_flow_hashfields_adjust
1798                                         (rss_desc, tunnel,
1799                                          MLX5_IPV6_LAYER_TYPES,
1800                                          MLX5_IPV6_IBV_RX_HASH);
1801                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1802                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1803                         break;
1804                 case RTE_FLOW_ITEM_TYPE_TCP:
1805                         flow_verbs_translate_item_tcp(dev_flow, items,
1806                                                       item_flags);
1807                         subpriority = MLX5_PRIORITY_MAP_L4;
1808                         dev_flow->hash_fields |=
1809                                 mlx5_flow_hashfields_adjust
1810                                         (rss_desc, tunnel, ETH_RSS_TCP,
1811                                          (IBV_RX_HASH_SRC_PORT_TCP |
1812                                           IBV_RX_HASH_DST_PORT_TCP));
1813                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1814                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1815                         break;
1816                 case RTE_FLOW_ITEM_TYPE_UDP:
1817                         flow_verbs_translate_item_udp(dev_flow, items,
1818                                                       item_flags);
1819                         subpriority = MLX5_PRIORITY_MAP_L4;
1820                         dev_flow->hash_fields |=
1821                                 mlx5_flow_hashfields_adjust
1822                                         (rss_desc, tunnel, ETH_RSS_UDP,
1823                                          (IBV_RX_HASH_SRC_PORT_UDP |
1824                                           IBV_RX_HASH_DST_PORT_UDP));
1825                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1826                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1827                         break;
1828                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1829                         flow_verbs_translate_item_vxlan(dev_flow, items,
1830                                                         item_flags);
1831                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1832                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1833                         break;
1834                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1835                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1836                                                             item_flags);
1837                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1838                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1839                         break;
1840                 case RTE_FLOW_ITEM_TYPE_GRE:
1841                         flow_verbs_translate_item_gre(dev_flow, items,
1842                                                       item_flags);
1843                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1844                         item_flags |= MLX5_FLOW_LAYER_GRE;
1845                         break;
1846                 case RTE_FLOW_ITEM_TYPE_MPLS:
1847                         flow_verbs_translate_item_mpls(dev_flow, items,
1848                                                        item_flags);
1849                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1850                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1851                         break;
1852                 default:
1853                         return rte_flow_error_set(error, ENOTSUP,
1854                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1855                                                   NULL,
1856                                                   "item not supported");
1857                 }
1858         }
1859         dev_flow->handle->layers = item_flags;
1860         /* Other members of attr will be ignored. */
1861         dev_flow->verbs.attr.priority =
1862                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1863         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1864         return 0;
1865 }
1866
1867 /**
1868  * Remove the flow from the NIC but keeps it in memory.
1869  *
1870  * @param[in] dev
1871  *   Pointer to the Ethernet device structure.
1872  * @param[in, out] flow
1873  *   Pointer to flow structure.
1874  */
1875 static void
1876 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1877 {
1878         struct mlx5_priv *priv = dev->data->dev_private;
1879         struct mlx5_flow_handle *handle;
1880         uint32_t handle_idx;
1881
1882         if (!flow)
1883                 return;
1884         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1885                        handle_idx, handle, next) {
1886                 if (handle->drv_flow) {
1887                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1888                         handle->drv_flow = NULL;
1889                 }
1890                 /* hrxq is union, don't touch it only the flag is set. */
1891                 if (handle->rix_hrxq) {
1892                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1893                                 mlx5_drop_action_destroy(dev);
1894                                 handle->rix_hrxq = 0;
1895                         } else if (handle->fate_action ==
1896                                    MLX5_FLOW_FATE_QUEUE) {
1897                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
1898                                 handle->rix_hrxq = 0;
1899                         }
1900                 }
1901                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1902                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1903         }
1904 }
1905
1906 /**
1907  * Remove the flow from the NIC and the memory.
1908  *
1909  * @param[in] dev
1910  *   Pointer to the Ethernet device structure.
1911  * @param[in, out] flow
1912  *   Pointer to flow structure.
1913  */
1914 static void
1915 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1916 {
1917         struct mlx5_priv *priv = dev->data->dev_private;
1918         struct mlx5_flow_handle *handle;
1919
1920         if (!flow)
1921                 return;
1922         flow_verbs_remove(dev, flow);
1923         while (flow->dev_handles) {
1924                 uint32_t tmp_idx = flow->dev_handles;
1925
1926                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1927                                    tmp_idx);
1928                 if (!handle)
1929                         return;
1930                 flow->dev_handles = handle->next.next;
1931                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1932                            tmp_idx);
1933         }
1934         if (flow->counter) {
1935                 flow_verbs_counter_release(dev, flow->counter);
1936                 flow->counter = 0;
1937         }
1938 }
1939
1940 /**
1941  * Apply the flow to the NIC.
1942  *
1943  * @param[in] dev
1944  *   Pointer to the Ethernet device structure.
1945  * @param[in, out] flow
1946  *   Pointer to flow structure.
1947  * @param[out] error
1948  *   Pointer to error structure.
1949  *
1950  * @return
1951  *   0 on success, a negative errno value otherwise and rte_errno is set.
1952  */
1953 static int
1954 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1955                  struct rte_flow_error *error)
1956 {
1957         struct mlx5_priv *priv = dev->data->dev_private;
1958         struct mlx5_flow_handle *handle;
1959         struct mlx5_flow *dev_flow;
1960         struct mlx5_hrxq *hrxq;
1961         uint32_t dev_handles;
1962         int err;
1963         int idx;
1964
1965         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1966                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1967                 handle = dev_flow->handle;
1968                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1969                         hrxq = mlx5_drop_action_create(dev);
1970                         if (!hrxq) {
1971                                 rte_flow_error_set
1972                                         (error, errno,
1973                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1974                                          "cannot get drop hash queue");
1975                                 goto error;
1976                         }
1977                 } else {
1978                         uint32_t hrxq_idx;
1979                         struct mlx5_flow_rss_desc *rss_desc =
1980                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
1981                                 [!!priv->flow_nested_idx];
1982
1983                         MLX5_ASSERT(rss_desc->queue_num);
1984                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
1985                                                  MLX5_RSS_HASH_KEY_LEN,
1986                                                  dev_flow->hash_fields,
1987                                                  rss_desc->queue,
1988                                                  rss_desc->queue_num);
1989                         if (!hrxq_idx)
1990                                 hrxq_idx = mlx5_hrxq_new
1991                                                 (dev, rss_desc->key,
1992                                                  MLX5_RSS_HASH_KEY_LEN,
1993                                                  dev_flow->hash_fields,
1994                                                  rss_desc->queue,
1995                                                  rss_desc->queue_num,
1996                                                  !!(handle->layers &
1997                                                  MLX5_FLOW_LAYER_TUNNEL));
1998                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1999                                               hrxq_idx);
2000                         if (!hrxq) {
2001                                 rte_flow_error_set
2002                                         (error, rte_errno,
2003                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2004                                          "cannot get hash queue");
2005                                 goto error;
2006                         }
2007                         handle->rix_hrxq = hrxq_idx;
2008                 }
2009                 MLX5_ASSERT(hrxq);
2010                 handle->drv_flow = mlx5_glue->create_flow
2011                                         (hrxq->qp, &dev_flow->verbs.attr);
2012                 if (!handle->drv_flow) {
2013                         rte_flow_error_set(error, errno,
2014                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2015                                            NULL,
2016                                            "hardware refuses to create flow");
2017                         goto error;
2018                 }
2019                 if (priv->vmwa_context &&
2020                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2021                         /*
2022                          * The rule contains the VLAN pattern.
2023                          * For VF we are going to create VLAN
2024                          * interface to make hypervisor set correct
2025                          * e-Switch vport context.
2026                          */
2027                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2028                 }
2029         }
2030         return 0;
2031 error:
2032         err = rte_errno; /* Save rte_errno before cleanup. */
2033         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2034                        dev_handles, handle, next) {
2035                 /* hrxq is union, don't touch it only the flag is set. */
2036                 if (handle->rix_hrxq) {
2037                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
2038                                 mlx5_drop_action_destroy(dev);
2039                                 handle->rix_hrxq = 0;
2040                         } else if (handle->fate_action ==
2041                                    MLX5_FLOW_FATE_QUEUE) {
2042                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
2043                                 handle->rix_hrxq = 0;
2044                         }
2045                 }
2046                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2047                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2048         }
2049         rte_errno = err; /* Restore rte_errno. */
2050         return -rte_errno;
2051 }
2052
2053 /**
2054  * Query a flow.
2055  *
2056  * @see rte_flow_query()
2057  * @see rte_flow_ops
2058  */
2059 static int
2060 flow_verbs_query(struct rte_eth_dev *dev,
2061                  struct rte_flow *flow,
2062                  const struct rte_flow_action *actions,
2063                  void *data,
2064                  struct rte_flow_error *error)
2065 {
2066         int ret = -EINVAL;
2067
2068         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2069                 switch (actions->type) {
2070                 case RTE_FLOW_ACTION_TYPE_VOID:
2071                         break;
2072                 case RTE_FLOW_ACTION_TYPE_COUNT:
2073                         ret = flow_verbs_counter_query(dev, flow, data, error);
2074                         break;
2075                 default:
2076                         return rte_flow_error_set(error, ENOTSUP,
2077                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2078                                                   actions,
2079                                                   "action not supported");
2080                 }
2081         }
2082         return ret;
2083 }
2084
2085 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2086         .validate = flow_verbs_validate,
2087         .prepare = flow_verbs_prepare,
2088         .translate = flow_verbs_translate,
2089         .apply = flow_verbs_apply,
2090         .remove = flow_verbs_remove,
2091         .destroy = flow_verbs_destroy,
2092         .query = flow_verbs_query,
2093 };